API tools faq
paste
Advertisement
Guest User

Untitled

a guest
Sep 29th, 2017
1,285
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 218.51 KB | None | 0 0
raw download clone embed print report
  1. {
  2. "configurations" : [
  3. {
  4. "hdfs-logsearch-conf" : {
  5. "properties_attributes" : { },
  6. "properties" : {
  7. "component_mappings" : "NAMENODE:hdfs_namenode;DATANODE:hdfs_datanode;SECONDARY_NAMENODE:hdfs_secondarynamenode;JOURNALNODE:hdfs_journalnode;ZKFC:hdfs_zkfc;NFS_GATEWAY:hdfs_nfs3",
  8. "content" : "\n{\n \"input\":[\n {\n \"type\":\"hdfs_datanode\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}/hadoop-{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}-datanode-*.log\"\n },\n {\n \"type\":\"hdfs_namenode\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}/hadoop-{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}-namenode-*.log\"\n },\n {\n \"type\":\"hdfs_journalnode\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}/hadoop-{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}-journalnode-*.log\"\n },\n {\n \"type\":\"hdfs_secondarynamenode\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}/hadoop-{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}-secondarynamenode-*.log\"\n },\n {\n \"type\":\"hdfs_zkfc\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}/hadoop-{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}-zkfc-*.log\"\n },\n {\n \"type\":\"hdfs_nfs3\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}/hadoop-{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}-nfs3-*.log\"\n },\n {\n \"type\":\"hdfs_audit\",\n \"rowtype\":\"audit\",\n \"is_enabled\":\"true\",\n \"add_fields\":{\n \"logType\":\"HDFSAudit\",\n \"enforcer\":\"hadoop-acl\",\n \"repoType\":\"1\",\n \"repo\":\"hdfs\"\n },\n \"path\":\"{{default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/hadoop-env/hdfs_user', 'hdfs')}}/hdfs-audit.log\"\n }\n ],\n \"filter\":[\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"hdfs_datanode\",\n \"hdfs_journalnode\",\n \"hdfs_secondarynamenode\",\n \"hdfs_namenode\",\n \"hdfs_zkfc\",\n \"hdfs_nfs3\"\n ]\n }\n },\n \"log4j_format\":\"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\\\(%{INT:line_number}\\\\)\\\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"target_date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n }\n }\n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"hdfs_audit\"\n ]\n }\n },\n \"log4j_format\":\"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:evtTime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:evtTime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"evtTime\":{\n \"map_date\":{\n \"target_date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n }\n }\n },\n {\n \"filter\":\"keyvalue\",\n \"sort_order\":1,\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"hdfs_audit\"\n ]\n }\n },\n \"source_field\":\"log_message\",\n \"value_split\":\"=\",\n \"field_split\":\"\\t\",\n \"post_map_values\":{\n \"src\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"resource\"\n }\n },\n \"ip\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"cliIP\"\n }\n },\n \"allowed\":[\n {\n \"map_fieldvalue\":{\n \"pre_value\":\"true\",\n \"post_value\":\"1\"\n }\n },\n {\n \"map_fieldvalue\":{\n \"pre_value\":\"false\",\n \"post_value\":\"0\"\n }\n },\n {\n \"map_fieldname\":{\n \"new_fieldname\":\"result\"\n }\n }\n ],\n \"cmd\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"action\"\n }\n },\n \"proto\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"cliType\"\n }\n },\n \"callerContext\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"req_caller_id\"\n }\n }\n }\n },\n {\n \"filter\":\"grok\",\n \"sort_order\":2,\n \"source_field\":\"ugi\",\n \"remove_source_field\":\"false\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"hdfs_audit\"\n ]\n }\n },\n \"message_pattern\":\"%{USERNAME:p_user}.+auth:%{USERNAME:p_authType}.+via %{USERNAME:k_user}.+auth:%{USERNAME:k_authType}|%{USERNAME:user}.+auth:%{USERNAME:authType}|%{USERNAME:x_user}\",\n \"post_map_values\":{\n \"user\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"reqUser\"\n }\n },\n \"x_user\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"reqUser\"\n }\n },\n \"p_user\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"reqUser\"\n }\n },\n \"k_user\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"proxyUsers\"\n }\n },\n \"p_authType\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"authType\"\n }\n },\n \"k_authType\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"proxyAuthType\"\n }\n }\n }\n }\n ]\n }",
  9. "service_name" : "HDFS"
  10. }
  11. }
  12. },
  13. {
  14. "hive-logsearch-conf" : {
  15. "properties_attributes" : { },
  16. "properties" : {
  17. "component_mappings" : "HIVE_METASTORE:hive_metastore;HIVE_SERVER:hive_hiveserver2;WEBHCAT_SERVER:webhcat_server",
  18. "content" : "\n{\n \"input\":[\n {\n \"type\":\"hive_hiveserver2\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/hive-env/hive_log_dir', '/var/log/hive')}}/hiveserver2.log\"\n },\n {\n \"type\":\"hive_metastore\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/hive-env/hive_log_dir', '/var/log/hive')}}/hivemetastore.log\"\n },\n {\n \"type\": \"webhcat_server\",\n \"rowntype\":\"service\",\n \"path\":\"{{default('configurations/hive-env/hcat_log_dir', '/var/log/webhcat')}}/webhcat.log\"\n }\n ],\n \"filter\":[\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"hive_hiveserver2\",\n \"hive_metastore\"\n ]\n }\n },\n \"log4j_format\":\"%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\\\[%{DATA:thread_name}\\\\]:%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\\\(%{INT:line_number}\\\\)\\\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"target_date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n }\n }\n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"webhcat_server\"\n ]\n }\n },\n \"log4j_format\":\" %-5p | %d{DATE} | %c | %m%n\",\n \"multiline_pattern\":\"^(%{SPACE}%{LOGLEVEL:level}%{CUSTOM_SEPARATOR}%{CUSTOM_DATESTAMP:logtime})\",\n \"message_pattern\":\"(?m)^%{SPACE}%{LOGLEVEL:level}%{CUSTOM_SEPARATOR}%{CUSTOM_DATESTAMP:logtime}%{CUSTOM_SEPARATOR}%{JAVACLASS:file}%{CUSTOM_SEPARATOR}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"target_date_pattern\":\"dd MMM yyyy HH:mm:ss,SSS\"\n }\n },\n \"level\":{\n \"map_fieldvalue\":{\n \"pre_value\":\"WARNING\",\n \"post_value\":\"WARN\"\n }\n }\n }\n }\n ]\n }",
  19. "service_name" : "Hive"
  20. }
  21. }
  22. },
  23. {
  24. "zoo.cfg" : {
  25. "properties_attributes" : { },
  26. "properties" : {
  27. "autopurge.purgeInterval" : "24",
  28. "dataDir" : "/hadoop/zookeeper",
  29. "autopurge.snapRetainCount" : "30",
  30. "clientPort" : "2181",
  31. "initLimit" : "10",
  32. "tickTime" : "3000",
  33. "syncLimit" : "5"
  34. }
  35. }
  36. },
  37. {
  38. "ams-hbase-site" : {
  39. "properties_attributes" : {
  40. "final" : {
  41. "hbase.zookeeper.quorum" : "true"
  42. }
  43. },
  44. "properties" : {
  45. "hfile.block.cache.size" : "0.3",
  46. "phoenix.query.spoolThresholdBytes" : "20971520",
  47. "hbase.zookeeper.property.dataDir" : "${hbase.tmp.dir}/zookeeper",
  48. "hbase.regionserver.thread.compaction.large" : "2",
  49. "hbase.master.port" : "61300",
  50. "zookeeper.session.timeout.localHBaseCluster" : "120000",
  51. "phoenix.query.keepAliveMs" : "300000",
  52. "hbase.local.dir" : "${hbase.tmp.dir}/local",
  53. "phoenix.query.timeoutMs" : "300000",
  54. "hbase.regionserver.thread.compaction.small" : "3",
  55. "hbase.zookeeper.quorum" : "{{zookeeper_quorum_hosts}}",
  56. "hbase.client.scanner.timeout.period" : "300000",
  57. "hbase.regionserver.info.port" : "61330",
  58. "phoenix.query.rowKeyOrderSaltedTable" : "true",
  59. "phoenix.mutate.batchSize" : "10000",
  60. "zookeeper.znode.parent" : "/ams-hbase-unsecure",
  61. "hbase.master.info.port" : "61310",
  62. "hbase.rootdir" : "file:///var/lib/ambari-metrics-collector/hbase",
  63. "hbase.cluster.distributed" : "false",
  64. "hbase.hregion.majorcompaction" : "0",
  65. "hbase.hstore.flusher.count" : "2",
  66. "hbase.master.normalizer.class" : "org.apache.hadoop.hbase.master.normalizer.SimpleRegionNormalizer",
  67. "hbase.snapshot.enabled" : "false",
  68. "phoenix.spool.directory" : "${hbase.tmp.dir}/phoenix-spool",
  69. "phoenix.coprocessor.maxMetaDataCacheSize" : "20480000",
  70. "hbase.zookeeper.property.tickTime" : "6000",
  71. "hbase.zookeeper.leaderport" : "61388",
  72. "phoenix.sequence.saltBuckets" : "2",
  73. "hbase.hstore.blockingStoreFiles" : "200",
  74. "zookeeper.session.timeout" : "120000",
  75. "phoenix.coprocessor.maxServerCacheTimeToLiveMs" : "60000",
  76. "hbase.master.info.bindAddress" : "0.0.0.0",
  77. "hbase.regionserver.global.memstore.lowerLimit" : "0.3",
  78. "dfs.client.read.shortcircuit" : "true",
  79. "phoenix.query.maxGlobalMemoryPercentage" : "15",
  80. "hbase.hregion.memstore.flush.size" : "134217728",
  81. "hbase.hregion.max.filesize" : "4294967296",
  82. "phoenix.groupby.maxCacheSize" : "307200000",
  83. "hbase.master.wait.on.regionservers.mintostart" : "1",
  84. "hbase.regionserver.global.memstore.upperLimit" : "0.35",
  85. "hbase.normalizer.period" : "600000",
  86. "hbase.tmp.dir" : "/var/lib/ambari-metrics-collector/hbase-tmp",
  87. "hbase.client.scanner.caching" : "10000",
  88. "hbase.replication" : "false",
  89. "hbase.rpc.timeout" : "300000",
  90. "hbase.superuser" : "activity_explorer,activity_analyzer",
  91. "hbase.zookeeper.peerport" : "61288",
  92. "hbase.regionserver.port" : "61320",
  93. "hbase.zookeeper.property.clientPort" : "{{zookeeper_clientPort}}",
  94. "hbase.hregion.memstore.block.multiplier" : "4",
  95. "hbase.normalizer.enabled" : "false"
  96. }
  97. }
  98. },
  99. {
  100. "zookeeper-log4j" : {
  101. "properties_attributes" : { },
  102. "properties" : {
  103. "zookeeper_log_max_backup_size" : "10",
  104. "zookeeper_log_number_of_backup_files" : "10",
  105. "content" : "\n#\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied. See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n#\n#\n\n#\n# ZooKeeper Logging Configuration\n#\n\n# DEFAULT: console appender only\nlog4j.rootLogger=INFO, CONSOLE, ROLLINGFILE\n\n# Example with rolling log file\n#log4j.rootLogger=DEBUG, CONSOLE, ROLLINGFILE\n\n# Example with rolling log file and tracing\n#log4j.rootLogger=TRACE, CONSOLE, ROLLINGFILE, TRACEFILE\n\n#\n# Log INFO level and above messages to the console\n#\nlog4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender\nlog4j.appender.CONSOLE.Threshold=INFO\nlog4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout\nlog4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n\n\n#\n# Add ROLLINGFILE to rootLogger to get log file output\n# Log DEBUG level and above messages to a log file\nlog4j.appender.ROLLINGFILE=org.apache.log4j.RollingFileAppender\nlog4j.appender.ROLLINGFILE.Threshold=DEBUG\nlog4j.appender.ROLLINGFILE.File={{zk_log_dir}}/zookeeper.log\n\n# Max log file size of 10MB\nlog4j.appender.ROLLINGFILE.MaxFileSize={{zookeeper_log_max_backup_size}}MB\n# uncomment the next line to limit number of backup files\n#log4j.appender.ROLLINGFILE.MaxBackupIndex={{zookeeper_log_number_of_backup_files}}\n\nlog4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout\nlog4j.appender.ROLLINGFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n\n\n\n#\n# Add TRACEFILE to rootLogger to get log file output\n# Log DEBUG level and above messages to a log file\nlog4j.appender.TRACEFILE=org.apache.log4j.FileAppender\nlog4j.appender.TRACEFILE.Threshold=TRACE\nlog4j.appender.TRACEFILE.File=zookeeper_trace.log\n\nlog4j.appender.TRACEFILE.layout=org.apache.log4j.PatternLayout\n### Notice we are including log4j's NDC here (%x)\nlog4j.appender.TRACEFILE.layout.ConversionPattern=%d{ISO8601} - %-5p [%t:%C{1}@%L][%x] - %m%n"
  106. }
  107. }
  108. },
  109. {
  110. "pig-env" : {
  111. "properties_attributes" : { },
  112. "properties" : {
  113. "content" : "\nJAVA_HOME={{java64_home}}\nHADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}\n\nif [ -d \"/usr/lib/tez\" ]; then\n PIG_OPTS=\"$PIG_OPTS -Dmapreduce.framework.name=yarn\"\nfi"
  114. }
  115. }
  116. },
  117. {
  118. "ranger-hdfs-policymgr-ssl" : {
  119. "properties_attributes" : { },
  120. "properties" : { }
  121. }
  122. },
  123. {
  124. "activity-log4j" : {
  125. "properties_attributes" : { },
  126. "properties" : {
  127. "activity_max_file_size" : "30",
  128. "activity-log4j-content" : "\n# Copyright (c) 2011-2017, Hortonworks Inc. All rights reserved.\n# Except as expressly permitted in a written agreement between you\n# or your company and Hortonworks, Inc, any use, reproduction,\n# modification, redistribution, sharing, lending or other exploitation\n# of all or any part of the contents of this file is strictly prohibited.\n\n# Define some default values that can be overridden by system properties\n# Root logger option\nlog4j.rootLogger=INFO,file\n\nlog4j.appender.file=org.apache.log4j.RollingFileAppender\nlog4j.appender.file.File={{activity_log_dir}}/${log.file.name}\nlog4j.appender.file.MaxFileSize={{activity_max_file_size}}MB\nlog4j.appender.file.MaxBackupIndex={{activity_max_backup_index}}\nlog4j.appender.file.layout=org.apache.log4j.PatternLayout\nlog4j.appender.file.layout.ConversionPattern=%d{ISO8601} %5p [%t] %c{1}:%L - %m%n",
  129. "activity_log_dir" : "/var/log/smartsense-activity",
  130. "activity_max_backup_index" : "10"
  131. }
  132. }
  133. },
  134. {
  135. "ams-logsearch-conf" : {
  136. "properties_attributes" : { },
  137. "properties" : {
  138. "component_mappings" : "METRICS_COLLECTOR:ams_collector,ams_hbase_master,ams_hbase_regionserver;METRICS_MONITOR:ams_monitor;METRICS_GRAFANA:ams_grafana",
  139. "content" : "\n{\n \"input\":[\n {\n \"type\":\"ams_hbase_master\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/ams-env/metrics_collector_log_dir', '/var/log/ambari-metrics-collector')}}/hbase-ams-master-*.log\"\n },\n {\n \"type\":\"ams_hbase_regionserver\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/ams-env/metrics_collector_log_dir', '/var/log/ambari-metrics-collector')}}/hbase-ams-regionserver-*.log\"\n },\n {\n \"type\":\"ams_collector\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/ams-env/metrics_collector_log_dir', '/var/log/ambari-metrics-collector')}}/ambari-metrics-collector.log\"\n },\n {\n \"type\":\"ams_monitor\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/ams-env/metrics_monitor_log_dir', '/var/log/ambari-metrics-monitor')}}/ambari-metrics-monitor.out\"\n },\n {\n \"type\":\"ams_grafana\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/ams-grafana-env/metrics_grafana_log_dir', '/var/log/ambari-metrics-grafana')}}/grafana.log\"\n }\n ],\n \"filter\":[\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"ams_collector\"\n ]\n }\n },\n \"log4j_format\":\"%d{ISO8601} %p %c: %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"target_date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n }\n }\n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"ams_hbase_master\",\n \"ams_hbase_regionserver\"\n ]\n }\n },\n \"log4j_format\":\"%d{ISO8601} %-5p [%t] %c{2}: %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\\\[%{DATA:thread_name}\\\\]%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"target_date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n }\n }\n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"ams_grafana\"\n ]\n }\n },\n \"log4j_format\":\"%d{ISO8601} %-5p [%t] %c{2}: %m%n\",\n \"multiline_pattern\":\"^(%{DATESTAMP:logtime})\",\n \"message_pattern\":\"(?m)^%{DATESTAMP:logtime}%{SPACE}\\\\[%{WORD:level}\\\\]%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"target_date_pattern\":\"yyyy/MM/dd HH:mm:ss\"\n }\n },\n \"level\":[\n {\n \"map_fieldvalue\":{\n \"pre_value\":\"I\",\n \"post_value\":\"INFO\"\n }\n },\n {\n \"map_fieldvalue\":{\n \"pre_value\":\"W\",\n \"post_value\":\"WARN\"\n }\n },\n {\n \"map_fieldvalue\":{\n \"pre_value\":\"D\",\n \"post_value\":\"DEBUG\"\n }\n },\n {\n \"map_fieldvalue\":{\n \"pre_value\":\"E\",\n \"post_value\":\"ERROR\"\n }\n },\n {\n \"map_fieldvalue\":{\n \"pre_value\":\"F\",\n \"post_value\":\"FATAL\"\n }\n }\n ]\n }\n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"ams_monitor\"\n ]\n }\n },\n \"log4j_format\":\"\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\\\[%{LOGLEVEL:level}\\\\]%{SPACE}%{JAVAFILE:file}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"target_date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n }\n },\n \"level\":[\n {\n \"map_fieldvalue\":{\n \"pre_value\":\"WARNING\",\n \"post_value\":\"WARN\"\n }\n }\n ]\n }\n ]\n }",
  140. "service_name" : "AMS"
  141. }
  142. }
  143. },
  144. {
  145. "ranger-yarn-policymgr-ssl" : {
  146. "properties_attributes" : { },
  147. "properties" : { }
  148. }
  149. },
  150. {
  151. "hive-log4j2" : {
  152. "properties_attributes" : { },
  153. "properties" : {
  154. "hive2_log_maxfilesize" : "256",
  155. "hive2_log_maxbackupindex" : "30",
  156. "content" : "\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nstatus = INFO\nname = HiveLog4j2\npackages = org.apache.hadoop.hive.ql.log\n\n# list of properties\nproperty.hive.log.level = {{hive_log_level}}\nproperty.hive.root.logger = DRFA\nproperty.hive.log.dir = ${sys:java.io.tmpdir}/${sys:user.name}\nproperty.hive.log.file = hive.log\n\n# list of all appenders\nappenders = console, DRFA\n\n# console appender\nappender.console.type = Console\nappender.console.name = console\nappender.console.target = SYSTEM_ERR\nappender.console.layout.type = PatternLayout\nappender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n\n\n# daily rolling file appender\nappender.DRFA.type = RollingFile\nappender.DRFA.name = DRFA\nappender.DRFA.fileName = ${sys:hive.log.dir}/${sys:hive.log.file}\n# Use %pid in the filePattern to append process-id@host-name to the filename if you want separate log files for different CLI session\nappender.DRFA.filePattern = ${sys:hive.log.dir}/${sys:hive.log.file}.%d{yyyy-MM-dd}_%i.gz\nappender.DRFA.layout.type = PatternLayout\nappender.DRFA.layout.pattern = %d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n\nappender.DRFA.policies.type = Policies\nappender.DRFA.policies.time.type = TimeBasedTriggeringPolicy\nappender.DRFA.policies.time.interval = 1\nappender.DRFA.policies.time.modulate = true\nappender.DRFA.strategy.type = DefaultRolloverStrategy\nappender.DRFA.strategy.max = {{hive2_log_maxbackupindex}}\nappender.DRFA.policies.fsize.type = SizeBasedTriggeringPolicy\nappender.DRFA.policies.fsize.size = {{hive2_log_maxfilesize}}MB\n\n# list of all loggers\nloggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX\n\nlogger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn\nlogger.NIOServerCnxn.level = WARN\n\nlogger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO\nlogger.ClientCnxnSocketNIO.level = WARN\n\nlogger.DataNucleus.name = DataNucleus\nlogger.DataNucleus.level = ERROR\n\nlogger.Datastore.name = Datastore\nlogger.Datastore.level = ERROR\n\nlogger.JPOX.name = JPOX\nlogger.JPOX.level = ERROR\n\n# root logger\nrootLogger.level = ${sys:hive.log.level}\nrootLogger.appenderRefs = root\nrootLogger.appenderRef.root.ref = ${sys:hive.root.logger}"
  157. }
  158. }
  159. },
  160. {
  161. "mapred-logsearch-conf" : {
  162. "properties_attributes" : { },
  163. "properties" : {
  164. "component_mappings" : "HISTORYSERVER:mapred_historyserver",
  165. "content" : "\n{\n \"input\":[\n {\n \"type\":\"mapred_historyserver\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/mapred-env/mapred_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/mapred-env/mapred_user', 'mapred')}}/mapred-{{default('configurations/mapred-env/mapred_user', 'mapred')}}-historyserver*.log\"\n }\n ],\n \"filter\":[\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"mapred_historyserver\"\n ]\n }\n },\n \"log4j_format\":\"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\\\(%{INT:line_number}\\\\)\\\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"target_date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n }\n }\n }\n ]\n }",
  166. "service_name" : "MapReduce"
  167. }
  168. }
  169. },
  170. {
  171. "ranger-hive-policymgr-ssl" : {
  172. "properties_attributes" : { },
  173. "properties" : { }
  174. }
  175. },
  176. {
  177. "zookeeper-logsearch-conf" : {
  178. "properties_attributes" : { },
  179. "properties" : {
  180. "component_mappings" : "ZOOKEEPER_SERVER:zookeeper",
  181. "content" : "\n{\n \"input\":[\n {\n \"type\":\"zookeeper\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/zookeeper-env/zk_log_dir', '/var/log/zookeeper')}}/zookeeper*.log\"\n }\n ],\n \"filter\":[\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\"type\":[\"zookeeper\"]}\n },\n \"log4j_format\":\"%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}-%{SPACE}%{LOGLEVEL:level}%{SPACE}\\\\[%{DATA:thread_name}\\\\@%{INT:line_number}\\\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\": {\n \"logtime\": {\n \"map_date\":{\n \"target_date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n }\n }\n }\n ]\n}",
  182. "service_name" : "Zookeeper"
  183. }
  184. }
  185. },
  186. {
  187. "hcat-env" : {
  188. "properties_attributes" : { },
  189. "properties" : {
  190. "content" : "\n # Licensed to the Apache Software Foundation (ASF) under one\n # or more contributor license agreements. See the NOTICE file\n # distributed with this work for additional information\n # regarding copyright ownership. The ASF licenses this file\n # to you under the Apache License, Version 2.0 (the\n # \"License\"); you may not use this file except in compliance\n # with the License. You may obtain a copy of the License at\n #\n # http://www.apache.org/licenses/LICENSE-2.0\n #\n # Unless required by applicable law or agreed to in writing, software\n # distributed under the License is distributed on an \"AS IS\" BASIS,\n # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n # See the License for the specific language governing permissions and\n # limitations under the License.\n\n JAVA_HOME={{java64_home}}\n HCAT_PID_DIR={{hcat_pid_dir}}/\n HCAT_LOG_DIR={{hcat_log_dir}}/\n HCAT_CONF_DIR={{hcat_conf_dir}}\n HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}\n #DBROOT is the path where the connector jars are downloaded\n DBROOT={{hcat_dbroot}}\n USER={{hcat_user}}\n METASTORE_PORT={{hive_metastore_port}}"
  191. }
  192. }
  193. },
  194. {
  195. "ssl-server" : {
  196. "properties_attributes" : { },
  197. "properties" : {
  198. "ssl.server.truststore.location" : "/etc/security/serverKeys/all.jks",
  199. "ssl.server.truststore.reload.interval" : "10000",
  200. "ssl.server.truststore.type" : "jks",
  201. "ssl.server.keystore.location" : "/etc/security/serverKeys/keystore.jks",
  202. "ssl.server.keystore.type" : "jks"
  203. }
  204. }
  205. },
  206. {
  207. "pig-properties" : {
  208. "properties_attributes" : { },
  209. "properties" : {
  210. "content" : "\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied. See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# Pig default configuration file. All values can be overwritten by pig.properties and command line arguments.\n# see bin/pig -help\n\n# brief logging (no timestamps)\nbrief=false\n\n# debug level, INFO is default\ndebug=INFO\n\n# verbose print all log messages to screen (default to print only INFO and above to screen)\nverbose=false\n\n# exectype local|mapreduce|tez, mapreduce is default\nexectype=tez\n\n# Enable insertion of information about script into hadoop job conf \npig.script.info.enabled=true\n\n# Do not spill temp files smaller than this size (bytes)\npig.spill.size.threshold=5000000\n\n# EXPERIMENT: Activate garbage collection when spilling a file bigger than this size (bytes)\n# This should help reduce the number of files being spilled.\npig.spill.gc.activation.size=40000000\n\n# the following two parameters are to help estimate the reducer number\npig.exec.reducers.bytes.per.reducer=1000000000\npig.exec.reducers.max=999\n\n# Temporary location to store the intermediate data.\npig.temp.dir=/tmp/\n\n# Threshold for merging FRJoin fragment files\npig.files.concatenation.threshold=100\npig.optimistic.files.concatenation=false;\n\npig.disable.counter=false\n\nhcat.bin=/usr/bin/hcat"
  211. }
  212. }
  213. },
  214. {
  215. "hst-agent-conf" : {
  216. "properties_attributes" : { },
  217. "properties" : {
  218. "security.anonymization.max.heap" : "2048",
  219. "agent.tmp_dir" : "/var/lib/smartsense/hst-agent/data/tmp",
  220. "upload.retry_count" : "100",
  221. "server.connection_retry_interval" : "10",
  222. "agent.version" : "1.4.2.2.5.2.0-298",
  223. "upload.retry_interval" : "15",
  224. "agent.loglevel" : "INFO",
  225. "bundle.logs_to_capture" : "(.*).log$,(.*).out$",
  226. "server.connection_retry_count" : "100"
  227. }
  228. }
  229. },
  230. {
  231. "ranger-hive-plugin-properties" : {
  232. "properties_attributes" : { },
  233. "properties" : { }
  234. }
  235. },
  236. {
  237. "ranger-yarn-security" : {
  238. "properties_attributes" : { },
  239. "properties" : { }
  240. }
  241. },
  242. {
  243. "hadoop-policy" : {
  244. "properties_attributes" : { },
  245. "properties" : {
  246. "security.inter.datanode.protocol.acl" : "*",
  247. "security.refresh.usertogroups.mappings.protocol.acl" : "hadoop",
  248. "security.admin.operations.protocol.acl" : "hadoop",
  249. "security.client.datanode.protocol.acl" : "*",
  250. "security.datanode.protocol.acl" : "*",
  251. "security.inter.tracker.protocol.acl" : "*",
  252. "security.job.client.protocol.acl" : "*",
  253. "security.client.protocol.acl" : "*",
  254. "security.job.task.protocol.acl" : "*",
  255. "security.refresh.policy.protocol.acl" : "hadoop",
  256. "security.namenode.protocol.acl" : "*"
  257. }
  258. }
  259. },
  260. {
  261. "ranger-yarn-audit" : {
  262. "properties_attributes" : { },
  263. "properties" : { }
  264. }
  265. },
  266. {
  267. "tez-interactive-site" : {
  268. "properties_attributes" : { },
  269. "properties" : {
  270. "tez.runtime.pipelined.sorter.lazy-allocate.memory" : "true",
  271. "tez.runtime.report.partition.stats" : "true",
  272. "tez.am.client.heartbeat.timeout.secs" : "90",
  273. "tez.runtime.shuffle.connect.timeout" : "30000",
  274. "tez.runtime.enable.final-merge.in.output" : "false",
  275. "tez.am.client.heartbeat.poll.interval.millis" : "6000",
  276. "tez.container.max.java.heap.fraction" : "-1",
  277. "tez.runtime.shuffle.memory.limit.percent" : "0.25",
  278. "tez.session.am.dag.submit.timeout.secs" : "1209600",
  279. "tez.runtime.shuffle.read.timeout" : "30000",
  280. "tez.am.am-rm.heartbeat.interval-ms.max" : "10000",
  281. "tez.lib.uris" : "/hdp/apps/${hdp.version}/tez_hive2/tez.tar.gz",
  282. "tez.am.task.listener.thread-count" : "1",
  283. "tez.runtime.shuffle.fetch.buffer.percent" : "0.6",
  284. "tez.task.timeout-ms" : "90000",
  285. "tez.am.node-blacklisting.enabled" : "false",
  286. "tez.history.logging.taskattempt-filters" : "SERVICE_BUSY,EXTERNAL_PREEMPTION",
  287. "tez.am.task.reschedule.higher.priority" : "false",
  288. "tez.runtime.unordered.output.buffer.size-mb" : "100",
  289. "tez.runtime.pipelined-shuffle.enabled" : "false",
  290. "tez.runtime.shuffle.fetch.verify-disk-checksum" : "false",
  291. "tez.runtime.shuffle.parallel.copies" : "8",
  292. "tez.dag.recovery.enabled" : "false",
  293. "tez.task.heartbeat.timeout.check-ms" : "15000",
  294. "tez.am.resource.memory.mb" : "1536",
  295. "tez.history.logging.timeline.num-dags-per-group" : "5",
  296. "tez.grouping.node.local.only" : "true",
  297. "tez.runtime.io.sort.mb" : "512",
  298. "tez.runtime.shuffle.keep-alive.enabled" : "true"
  299. }
  300. }
  301. },
  302. {
  303. "anonymization-rules" : {
  304. "properties_attributes" : { },
  305. "properties" : {
  306. "anonymization-rules-content" : "{\n\"rules\":[\n {\n \"name\": \"IP\",\n \"rule_id\": \"Pattern\",\n \"patterns\": [\"(?<![a-z0-9\\\\.])[0-9]{1,3}\\\\.[0-9]{1,3}\\\\.[0-9]{1,3}\\\\.[0-9]{1,3}(?![a-z0-9\\\\.\\\\-])(?!\\\\.[0-9])\"],\n \"exclude_files\": [\"hdp-select*.*\", \"*version.txt\"]\n },\n {\n \"name\": \"IP_HDFS\",\n \"rule_id\": \"Pattern\",\n \"patterns\": [\"\\\\-[0-9]{1,3}\\\\.[0-9]{1,3}\\\\.[0-9]{1,3}\\\\.[0-9]{1,3}\\\\-\"],\n \"extract\": \"([0-9]{1,3}\\\\.[0-9]{1,3}\\\\.[0-9]{1,3}\\\\.[0-9]{1,3})\",\n \"include_files\": [\"hdfs_datanode_jmx.json\"]\n },\n {\n \"name\": \"DOMAIN_NAME\",\n \"rule_id\": \"Domain\"\n },\n {\n \"name\": \"SSN\",\n \"rule_id\": \"Pattern\",\n \"patterns\": [\"(?<![0-9x])([0-9x]{3}-[0-9x]{2}-[0-9]{4})(?![0-9x])\"],\n \"include_files\": [\"*.log*\", \"*.out*\"],\n \"exclude_files\" : [\"hst.log*\", \"hst.out*\"],\n \"shared\": false\n },\n {\n \"name\": \"CCN\",\n \"rule_id\": \"Pattern\",\n \"patterns\": [\"(?<![0-9x])(18|21|3[04678]|4[0-9x]|5[1-5]|60|65)[0-9x]{2}[- ]([0-9x]{4}[- ]){2}[0-9]{0,4}(?![0-9x])\"],\n \"extract\": \"(?<![0-9x])([0-9x -]+)(?![0-9x])\",\n \"include_files\": [\"*.log*\", \"*.out*\"],\n \"exclude_files\" : [\"hst.log.*\", \"hst.out\"],\n \"shared\": false\n },\n {\n \"name\": \"EMAIL\",\n \"rule_id\": \"Pattern\",\n \"patterns\": [\"(?<![a-z0-9._%+-])[a-z0-9._%+-]+@[a-z0-9.-]+\\\\.[a-z]{2,6}(?![a-z0-9._%+-])$?\"],\n \"exclude_files\" : [\"metadata.json\"],\n \"shared\": false\n },\n {\n \"name\": \"CORE_SITE_S3A\",\n \"rule_id\": \"Property\",\n \"properties\": [\"fs.s3a.session.token\",\"fs.s3a.proxy.host\",\"fs.s3a.proxy.username\"],\n \"include_files\": [\"core-site.xml\"],\n \"action\" : \"REPLACE\",\n \"replace_value\": \"Hidden\"\n },\n {\n \"name\": \"PASSWORDS\",\n \"rule_id\": \"Property\",\n \"properties\": [\".*password.*\", \".*awsAccessKeyId.*\", \".*awsSecretAccessKey.*\", \".*awsAccessKeyId.*\", \"fs.azure.account.key.*\", \"ranger.service.https.attrib.keystore.pass\"],\n \"include_files\": [\"*.xml\", \"*.properties\", \"*.yaml\", \"*.ini\"],\n \"exclude_files\" : [\"capacity-scheduler.xml\"],\n \"shared\": false\n },\n {\n \"name\": \"KNOX LDAP Password\",\n \"rule_id\": \"Property\",\n \"properties\": [\"main.ldapRealm.contextFactory.systemPassword\"],\n \"include_files\": [\"topologies/*.xml\"],\n \"parentNode\": \"param\",\n \"shared\": false\n },\n {\n \"name\": \"KEYSTORE_PASSWORD_LOGGED\",\n \"rule_id\": \"Pattern\",\n \"patterns\": [\"oozie.https.keystore.pass=([^\\\\s]*)\", \"OOZIE_HTTPS_KEYSTORE_PASS=([^\\\\s]*)\", \"ganglia_password=([^\\\\s]*)\", \"javax.jdo.option.ConnectionPassword=([^\\\\s]*)\"],\n \"extract\": \"=([^\\\\s]*)\",\n \"include_files\": [\"java_process.txt\", \"pid.txt\", \"ambari-agent.log\", \"oozie-env.cmd\", \"hive_set_v.txt\", \"beeline_set_v.txt\", \"process_list.txt\"],\n \"shared\": false\n },\n {\n \"name\": \"KEYSTORE_CONFIG\",\n \"rule_id\": \"Property\",\n \"properties\": [\"https.attrib.keystorePass\", \"HTTPS_KEYSTORE_PASS\"],\n \"include_files\": [\"ranger_webserver.properties\"],\n \"shared\": false\n },\n {\n \"name\": \"KMS_KEYSTORE_PASSWORD\",\n \"rule_id\": \"Pattern\",\n \"patterns\": [\"KMS_SSL_KEYSTORE_PASS=([^\\\\s]*)\"],\n \"extract\": \"=([^\\\\s]*)\",\n \"include_files\": [\"kms-env.sh\"],\n \"shared\": false\n },\n {\n \"name\": \"MAC_ADDRESS\",\n \"rule_id\": \"Pattern\",\n \"patterns\": [\"(([0-9a-f]{2}[:-]){5}[0-9a-f]{2})\"],\n \"extract\": \"([0-9a-f:-]{17})\",\n \"include_files\": [\"network_info.txt\"],\n \"shared\": true\n },\n {\n \"name\":\"IPV6\",\n \"rule_id\": \"Pattern\",\n \"patterns\": [\"inet6 addr:\\\\s((([\\\\da-f:\\\\/\\\\d]))*)\"],\n \"extract\": \":\\\\s((([\\\\da-f:\\\\/\\\\d]))*)\",\n \"include_files\": [\"network_info.txt\"],\n \"shared\":true\n },\n {\n \"name\": \"INTERPRETER_PASSWORDS\",\n \"rule_id\": \"Pattern\",\n \"patterns\": [\".*password.*:\\\\s*\\\"(.*)\\\"\"],\n \"extract\": \":\\\\s*\\\"(\\\\s|.*)\\\"\",\n \"include_files\": [\"interpreter.json\"],\n \"shared\": false\n }\n ]\n}"
  307. }
  308. }
  309. },
  310. {
  311. "core-site" : {
  312. "properties_attributes" : {
  313. "final" : {
  314. "fs.defaultFS" : "true"
  315. }
  316. },
  317. "properties" : {
  318. "fs.defaultFS" : "hdfs://%HOSTGROUP::host_group_1%:8020",
  319. "ipc.server.tcpnodelay" : "true",
  320. "mapreduce.jobtracker.webinterface.trusted" : "false",
  321. "hadoop.security.auth_to_local" : "DEFAULT",
  322. "hadoop.proxyuser.root.groups" : "*",
  323. "ipc.client.idlethreshold" : "8000",
  324. "hadoop.proxyuser.hdfs.groups" : "*",
  325. "fs.trash.interval" : "360",
  326. "hadoop.http.authentication.simple.anonymous.allowed" : "true",
  327. "hadoop.security.authorization" : "false",
  328. "ipc.client.connection.maxidletime" : "30000",
  329. "hadoop.proxyuser.hcat.groups" : "*",
  330. "hadoop.proxyuser.hive.hosts" : "%HOSTGROUP::host_group_1%",
  331. "hadoop.proxyuser.root.hosts" : "sandbox-hdp.hortonworks.com",
  332. "ha.failover-controller.active-standby-elector.zk.op.retries" : "120",
  333. "hadoop.security.authentication" : "simple",
  334. "hadoop.proxyuser.hdfs.hosts" : "*",
  335. "ipc.client.connect.max.retries" : "50",
  336. "io.file.buffer.size" : "131072",
  337. "hadoop.proxyuser.hcat.hosts" : "%HOSTGROUP::host_group_1%",
  338. "net.topology.script.file.name" : "/etc/hadoop/conf/topology_script.py",
  339. "io.compression.codecs" : "org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec",
  340. "hadoop.custom-extensions.root" : "/hdp/ext/{{major_stack_version}}/hadoop",
  341. "hadoop.proxyuser.hive.groups" : "*",
  342. "io.serializations" : "org.apache.hadoop.io.serializer.WritableSerialization"
  343. }
  344. }
  345. },
  346. {
  347. "hive-exec-log4j" : {
  348. "properties_attributes" : { },
  349. "properties" : {
  350. "content" : "\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# Define some default values that can be overridden by system properties\n\nhive.log.threshold=ALL\nhive.root.logger={{hive_log_level}},FA\nhive.log.dir=${java.io.tmpdir}/${user.name}\nhive.query.id=hadoop\nhive.log.file=${hive.query.id}.log\n\n# Define the root logger to the system property \"hadoop.root.logger\".\nlog4j.rootLogger=${hive.root.logger}, EventCounter\n\n# Logging Threshold\nlog4j.threshhold=${hive.log.threshold}\n\n#\n# File Appender\n#\n\nlog4j.appender.FA=org.apache.log4j.FileAppender\nlog4j.appender.FA.File=${hive.log.dir}/${hive.log.file}\nlog4j.appender.FA.layout=org.apache.log4j.PatternLayout\n\n# Pattern format: Date LogLevel LoggerName LogMessage\n#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\n# Debugging Pattern format\nlog4j.appender.FA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\n\n\n#\n# console\n# Add \"console\" to rootlogger above if you want to use this\n#\n\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\nlog4j.appender.console.target=System.err\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\nlog4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n\n\n#custom logging levels\n#log4j.logger.xxx=DEBUG\n\n#\n# Event Counter Appender\n# Sends counts of logging messages at different severity levels to Hadoop Metrics.\n#\nlog4j.appender.EventCounter=org.apache.hadoop.hive.shims.HiveEventCounter\n\n\nlog4j.category.DataNucleus=ERROR,FA\nlog4j.category.Datastore=ERROR,FA\nlog4j.category.Datastore.Schema=ERROR,FA\nlog4j.category.JPOX.Datastore=ERROR,FA\nlog4j.category.JPOX.Plugin=ERROR,FA\nlog4j.category.JPOX.MetaData=ERROR,FA\nlog4j.category.JPOX.Query=ERROR,FA\nlog4j.category.JPOX.General=ERROR,FA\nlog4j.category.JPOX.Enhancer=ERROR,FA\n\n\n# Silence useless ZK logs\nlog4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN,FA\nlog4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN,FA"
  351. }
  352. }
  353. },
  354. {
  355. "webhcat-env" : {
  356. "properties_attributes" : { },
  357. "properties" : {
  358. "content" : "\n# The file containing the running pid\nPID_FILE={{webhcat_pid_file}}\n\nTEMPLETON_LOG_DIR={{templeton_log_dir}}/\n\n\nWEBHCAT_LOG_DIR={{templeton_log_dir}}/\n\n# The console error log\nERROR_LOG={{templeton_log_dir}}/webhcat-console-error.log\n\n# The console log\nCONSOLE_LOG={{templeton_log_dir}}/webhcat-console.log\n\n#TEMPLETON_JAR=templeton_jar_name\n\n#HADOOP_PREFIX=hadoop_prefix\n\n#HCAT_PREFIX=hive_prefix\n\n# Set HADOOP_HOME to point to a specific hadoop install directory\nexport HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}"
  359. }
  360. }
  361. },
  362. {
  363. "hdfs-site" : {
  364. "properties_attributes" : {
  365. "final" : {
  366. "dfs.webhdfs.enabled" : "true",
  367. "dfs.namenode.http-address" : "true",
  368. "dfs.support.append" : "true",
  369. "dfs.namenode.name.dir" : "true",
  370. "dfs.datanode.failed.volumes.tolerated" : "true",
  371. "dfs.datanode.data.dir" : "true"
  372. }
  373. },
  374. "properties" : {
  375. "dfs.replication" : "3",
  376. "dfs.namenode.audit.log.async" : "true",
  377. "dfs.namenode.checkpoint.dir" : "/hadoop/hdfs/namesecondary",
  378. "dfs.namenode.avoid.read.stale.datanode" : "true",
  379. "manage.include.files" : "false",
  380. "dfs.journalnode.http-address" : "0.0.0.0:8480",
  381. "nfs.file.dump.dir" : "/tmp/.hdfs-nfs",
  382. "dfs.namenode.rpc-address" : "%HOSTGROUP::host_group_1%:8020",
  383. "dfs.namenode.https-address" : "%HOSTGROUP::host_group_1%:50470",
  384. "dfs.encrypt.data.transfer.cipher.suites" : "AES/CTR/NoPadding",
  385. "dfs.client.read.shortcircuit.streams.cache.size" : "4096",
  386. "dfs.hosts.exclude" : "/etc/hadoop/conf/dfs.exclude",
  387. "dfs.namenode.accesstime.precision" : "0",
  388. "dfs.namenode.fslock.fair" : "false",
  389. "dfs.permissions.enabled" : "true",
  390. "dfs.datanode.balance.bandwidthPerSec" : "6250000",
  391. "dfs.namenode.stale.datanode.interval" : "30000",
  392. "dfs.content-summary.limit" : "5000",
  393. "dfs.http.policy" : "HTTP_ONLY",
  394. "dfs.journalnode.https-address" : "0.0.0.0:8481",
  395. "dfs.datanode.du.reserved" : "8421447168",
  396. "dfs.domain.socket.path" : "/var/lib/hadoop-hdfs/dn_socket",
  397. "dfs.datanode.ipc.address" : "0.0.0.0:8010",
  398. "dfs.cluster.administrators" : " hdfs",
  399. "dfs.datanode.max.transfer.threads" : "4096",
  400. "dfs.namenode.handler.count" : "100",
  401. "dfs.https.port" : "50470",
  402. "dfs.replication.max" : "50",
  403. "dfs.client.read.shortcircuit" : "true",
  404. "dfs.webhdfs.enabled" : "true",
  405. "dfs.namenode.http-address" : "%HOSTGROUP::host_group_1%:50070",
  406. "dfs.namenode.name.dir" : "/hadoop/hdfs/namenode",
  407. "dfs.namenode.avoid.write.stale.datanode" : "true",
  408. "dfs.datanode.https.address" : "0.0.0.0:50475",
  409. "dfs.datanode.failed.volumes.tolerated" : "0",
  410. "dfs.client.retry.policy.enabled" : "false",
  411. "hadoop.caller.context.enabled" : "true",
  412. "dfs.namenode.startup.delay.block.deletion.sec" : "3600",
  413. "dfs.block.access.token.enable" : "true",
  414. "dfs.datanode.data.dir" : "/hadoop/hdfs/data",
  415. "dfs.permissions.superusergroup" : "hdfs",
  416. "dfs.blocksize" : "134217728",
  417. "dfs.namenode.checkpoint.edits.dir" : "${dfs.namenode.checkpoint.dir}",
  418. "nfs.exports.allowed.hosts" : "* rw",
  419. "dfs.datanode.address" : "0.0.0.0:50010",
  420. "dfs.blockreport.initialDelay" : "120",
  421. "dfs.datanode.data.dir.perm" : "750",
  422. "dfs.namenode.write.stale.datanode.ratio" : "1.0f",
  423. "dfs.namenode.name.dir.restore" : "true",
  424. "dfs.heartbeat.interval" : "3",
  425. "dfs.namenode.secondary.http-address" : "%HOSTGROUP::host_group_1%:50090",
  426. "dfs.namenode.checkpoint.txns" : "1000000",
  427. "dfs.journalnode.edits.dir" : "/hadoop/hdfs/journalnode",
  428. "dfs.support.append" : "true",
  429. "fs.permissions.umask-mode" : "022",
  430. "dfs.namenode.safemode.threshold-pct" : "1",
  431. "dfs.namenode.checkpoint.period" : "21600",
  432. "dfs.datanode.http.address" : "0.0.0.0:50075"
  433. }
  434. }
  435. },
  436. {
  437. "yarn-site" : {
  438. "properties_attributes" : { },
  439. "properties" : {
  440. "yarn.resourcemanager.zk-address" : "%HOSTGROUP::host_group_1%:2181",
  441. "yarn.nodemanager.container-monitor.interval-ms" : "3000",
  442. "yarn.timeline-service.entity-group-fs-store.active-dir" : "/ats/active/",
  443. "yarn.log.server.web-service.url" : "http://%HOSTGROUP::host_group_1%:8188/ws/v1/applicationhistory",
  444. "yarn.nodemanager.linux-container-executor.group" : "hadoop",
  445. "yarn.client.failover-proxy-provider" : "org.apache.hadoop.yarn.client.RequestHedgingRMFailoverProxyProvider",
  446. "yarn.application.classpath" : "/etc/hadoop/conf,/usr/hdp/current/hadoop-client/*,/usr/hdp/current/hadoop-client/lib/*,/usr/hdp/current/hadoop-hdfs-client/*,/usr/hdp/current/hadoop-hdfs-client/lib/*,/usr/hdp/current/hadoop-yarn-client/*,/usr/hdp/current/hadoop-yarn-client/lib/*,/usr/hdp/current/ext/hadoop/*",
  447. "yarn.nodemanager.kill-escape.launch-command-line" : "slider-agent,LLAP",
  448. "yarn.admin.acl" : "activity_analyzer,yarn",
  449. "yarn.timeline-service.entity-group-fs-store.cleaner-interval-seconds" : "3600",
  450. "yarn.nodemanager.remote-app-log-dir-suffix" : "logs",
  451. "yarn.nodemanager.address" : "0.0.0.0:45454",
  452. "yarn.resourcemanager.webapp.address" : "%HOSTGROUP::host_group_1%:8088",
  453. "yarn.timeline-service.leveldb-timeline-store.ttl-interval-ms" : "300000",
  454. "yarn.scheduler.maximum-allocation-vcores" : "3",
  455. "yarn.timeline-service.address" : "%HOSTGROUP::host_group_1%:10200",
  456. "yarn.timeline-service.enabled" : "true",
  457. "yarn.nodemanager.aux-services" : "mapreduce_shuffle,spark_shuffle,spark2_shuffle",
  458. "yarn.nodemanager.resource.cpu-vcores" : "3",
  459. "yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled" : "false",
  460. "yarn.nodemanager.aux-services.spark_shuffle.class" : "org.apache.spark.network.yarn.YarnShuffleService",
  461. "yarn.nodemanager.log-aggregation.debug-enabled" : "false",
  462. "yarn.resourcemanager.zk-acl" : "world:anyone:rwcda",
  463. "yarn.client.nodemanager-connect.max-wait-ms" : "60000",
  464. "yarn.http.policy" : "HTTP_ONLY",
  465. "yarn.timeline-service.http-authentication.simple.anonymous.allowed" : "true",
  466. "yarn.resourcemanager.zk-state-store.parent-path" : "/rmstore",
  467. "yarn.timeline-service.leveldb-timeline-store.start-time-read-cache-size" : "10000",
  468. "yarn.resourcemanager.admin.address" : "%HOSTGROUP::host_group_1%:8141",
  469. "yarn.nodemanager.linux-container-executor.cgroups.strict-resource-usage" : "false",
  470. "yarn.resourcemanager.resource-tracker.address" : "%HOSTGROUP::host_group_1%:8025",
  471. "yarn.nodemanager.delete.debug-delay-sec" : "0",
  472. "yarn.resourcemanager.fs.state-store.retry-policy-spec" : "2000, 500",
  473. "yarn.timeline-service.http-authentication.type" : "simple",
  474. "yarn.timeline-service.ttl-enable" : "true",
  475. "yarn.timeline-service.entity-group-fs-store.retain-seconds" : "604800",
  476. "yarn.log-aggregation.retain-seconds" : "2592000",
  477. "yarn.nodemanager.resource.memory-mb" : "6144",
  478. "yarn.nodemanager.disk-health-checker.min-healthy-disks" : "0.25",
  479. "yarn.node-labels.enabled" : "false",
  480. "yarn.nodemanager.kill-escape.user" : "hive",
  481. "yarn.resourcemanager.connect.max-wait.ms" : "-1",
  482. "yarn.resourcemanager.zk-num-retries" : "1000",
  483. "yarn.scheduler.minimum-allocation-vcores" : "1",
  484. "yarn.nodemanager.aux-services.spark2_shuffle.classpath" : "{{stack_root}}/${hdp.version}/spark2/aux/*",
  485. "yarn.resourcemanager.scheduler.class" : "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler",
  486. "yarn.resourcemanager.system-metrics-publisher.enabled" : "true",
  487. "yarn.timeline-service.client.max-retries" : "30",
  488. "yarn.timeline-service.client.retry-interval-ms" : "1000",
  489. "yarn.timeline-service.http-authentication.proxyuser.root.hosts" : "sandbox-hdp.hortonworks.com",
  490. "yarn.timeline-service.store-class" : "org.apache.hadoop.yarn.server.timeline.EntityGroupFSTimelineStore",
  491. "yarn.timeline-service.bind-host" : "0.0.0.0",
  492. "yarn.nodemanager.container-metrics.unregister-delay-ms" : "60000",
  493. "yarn.timeline-service.generic-application-history.store-class" : "org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore",
  494. "yarn.resourcemanager.fs.state-store.uri" : " ",
  495. "yarn.resourcemanager.recovery.enabled" : "true",
  496. "yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath" : "",
  497. "yarn.resourcemanager.scheduler.address" : "%HOSTGROUP::host_group_1%:8030",
  498. "yarn.timeline-service.client.fd-flush-interval-secs" : "5",
  499. "yarn.timeline-service.leveldb-timeline-store.start-time-write-cache-size" : "10000",
  500. "yarn.timeline-service.ttl-ms" : "2678400000",
  501. "yarn.nodemanager.health-checker.script.timeout-ms" : "60000",
  502. "yarn.nodemanager.aux-services.spark2_shuffle.class" : "org.apache.spark.network.yarn.YarnShuffleService",
  503. "yarn.log.server.url" : "http://%HOSTGROUP::host_group_1%:19888/jobhistory/logs",
  504. "yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes" : "org.apache.tez.dag.history.logging.ats.TimelineCachePluginImpl",
  505. "yarn.resourcemanager.address" : "%HOSTGROUP::host_group_1%:8050",
  506. "yarn.timeline-service.recovery.enabled" : "true",
  507. "yarn.nodemanager.recovery.dir" : "{{yarn_log_dir_prefix}}/nodemanager/recovery-state",
  508. "yarn.resourcemanager.scheduler.monitor.enable" : "false",
  509. "hadoop.registry.rm.enabled" : "true",
  510. "yarn.resourcemanager.bind-host" : "0.0.0.0",
  511. "yarn.nodemanager.log-aggregation.num-log-files-per-app" : "336",
  512. "yarn.nodemanager.admin-env" : "MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX",
  513. "yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size" : "10",
  514. "yarn.acl.enable" : "false",
  515. "manage.include.files" : "false",
  516. "yarn.resourcemanager.ha.enabled" : "false",
  517. "yarn.timeline-service.entity-group-fs-store.app-cache-size" : "10",
  518. "yarn.client.nodemanager-connect.retry-interval-ms" : "10000",
  519. "yarn.timeline-service.version" : "1.5",
  520. "yarn.timeline-service.leveldb-timeline-store.read-cache-size" : "104857600",
  521. "yarn.resourcemanager.monitor.capacity.preemption.natural_termination_factor" : "1",
  522. "yarn.nodemanager.local-dirs" : "/hadoop/yarn/local",
  523. "yarn.timeline-service.leveldb-timeline-store.path" : "/hadoop/yarn/timeline",
  524. "yarn.nodemanager.recovery.enabled" : "true",
  525. "yarn.resourcemanager.zk-timeout-ms" : "10000",
  526. "yarn.resourcemanager.am.max-attempts" : "2",
  527. "yarn.resourcemanager.state-store.max-completed-applications" : "${yarn.resourcemanager.max-completed-applications}",
  528. "yarn.node-labels.fs-store.root-dir" : "/system/yarn/node-labels",
  529. "yarn.node-labels.fs-store.retry-policy-spec" : "2000, 500",
  530. "yarn.nodemanager.aux-services.mapreduce_shuffle.class" : "org.apache.hadoop.mapred.ShuffleHandler",
  531. "yarn.log-aggregation-enable" : "true",
  532. "yarn.resourcemanager.work-preserving-recovery.enabled" : "true",
  533. "yarn.nodemanager.aux-services.spark_shuffle.classpath" : "{{stack_root}}/${hdp.version}/spark/aux/*",
  534. "yarn.resourcemanager.store.class" : "org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore",
  535. "yarn.timeline-service.http-authentication.proxyuser.root.groups" : "*",
  536. "yarn.timeline-service.leveldb-state-store.path" : "/hadoop/yarn/timeline",
  537. "yarn.nodemanager.log.retain-seconds" : "604800",
  538. "yarn.timeline-service.entity-group-fs-store.done-dir" : "/ats/done/",
  539. "yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage" : "90",
  540. "hadoop.registry.zk.quorum" : "%HOSTGROUP::host_group_1%:2181",
  541. "yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds" : "3600",
  542. "yarn.nodemanager.container-executor.class" : "org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor",
  543. "yarn.resourcemanager.nodes.exclude-path" : "/etc/hadoop/conf/yarn.exclude",
  544. "yarn.timeline-service.state-store-class" : "org.apache.hadoop.yarn.server.timeline.recovery.LeveldbTimelineStateStore",
  545. "yarn.nodemanager.bind-host" : "0.0.0.0",
  546. "yarn.resourcemanager.hostname" : "%HOSTGROUP::host_group_1%",
  547. "yarn.resourcemanager.monitor.capacity.preemption.total_preemption_per_round" : "1",
  548. "yarn.resourcemanager.connect.retry-interval.ms" : "15000",
  549. "yarn.timeline-service.webapp.address" : "%HOSTGROUP::host_group_1%:8188",
  550. "yarn.scheduler.minimum-allocation-mb" : "1024",
  551. "yarn.timeline-service.entity-group-fs-store.summary-store" : "org.apache.hadoop.yarn.server.timeline.RollingLevelDBTimelineStore",
  552. "yarn.nodemanager.health-checker.interval-ms" : "135000",
  553. "yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb" : "1000",
  554. "yarn.resourcemanager.zk-retry-interval-ms" : "1000",
  555. "yarn.scheduler.capacity.ordering-policy.priority-utilization.underutilized-preemption.enabled" : "false",
  556. "yarn.nodemanager.remote-app-log-dir" : "/app-logs",
  557. "yarn.scheduler.maximum-allocation-mb" : "6144",
  558. "yarn.nodemanager.vmem-check-enabled" : "false",
  559. "yarn.timeline-service.entity-group-fs-store.scan-interval-seconds" : "15",
  560. "yarn.resourcemanager.webapp.https.address" : "%HOSTGROUP::host_group_1%:8090",
  561. "yarn.timeline-service.webapp.https.address" : "%HOSTGROUP::host_group_1%:8190",
  562. "yarn.nodemanager.resource.percentage-physical-cpu-limit" : "80",
  563. "yarn.nodemanager.vmem-pmem-ratio" : "2.1",
  564. "yarn.nodemanager.log-aggregation.compression-type" : "gz",
  565. "yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms" : "10000",
  566. "yarn.nodemanager.log-dirs" : "/hadoop/yarn/log"
  567. }
  568. }
  569. },
  570. {
  571. "hiveserver2-site" : {
  572. "properties_attributes" : { },
  573. "properties" : {
  574. "hive.service.metrics.hadoop2.component" : "hiveserver2",
  575. "hive.security.authorization.enabled" : "false",
  576. "hive.metastore.metrics.enabled" : "true",
  577. "hive.service.metrics.reporter" : "HADOOP2"
  578. }
  579. }
  580. },
  581. {
  582. "pig-log4j" : {
  583. "properties_attributes" : { },
  584. "properties" : {
  585. "content" : "\n#\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied. See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n#\n#\n\n# ***** Set root logger level to DEBUG and its only appender to A.\nlog4j.logger.org.apache.pig=info, A\n\n# ***** A is set to be a ConsoleAppender.\nlog4j.appender.A=org.apache.log4j.ConsoleAppender\n# ***** A uses PatternLayout.\nlog4j.appender.A.layout=org.apache.log4j.PatternLayout\nlog4j.appender.A.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n"
  586. }
  587. }
  588. },
  589. {
  590. "hive-log4j" : {
  591. "properties_attributes" : { },
  592. "properties" : {
  593. "hive_log_maxbackupindex" : "30",
  594. "hive_log_maxfilesize" : "256",
  595. "content" : "\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# Define some default values that can be overridden by system properties\nhive.log.threshold=ALL\nhive.root.logger={{hive_log_level}},DRFA\nhive.log.dir=${java.io.tmpdir}/${user.name}\nhive.log.file=hive.log\n\n# Define the root logger to the system property \"hadoop.root.logger\".\nlog4j.rootLogger=${hive.root.logger}, EventCounter\n\n# Logging Threshold\nlog4j.threshold=${hive.log.threshold}\n\n#\n# Daily Rolling File Appender\n#\n# Use the PidDailyerRollingFileAppend class instead if you want to use separate log files\n# for different CLI session.\n#\n# log4j.appender.DRFA=org.apache.hadoop.hive.ql.log.PidDailyRollingFileAppender\n\nlog4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender\n\nlog4j.appender.DRFA.File=${hive.log.dir}/${hive.log.file}\n\n# Rollver at midnight\nlog4j.appender.DRFA.DatePattern=.yyyy-MM-dd\n\n# 30-day backup\n#log4j.appender.DRFA.MaxBackupIndex= {{hive_log_maxbackupindex}}\nlog4j.appender.DRFA.MaxFileSize = {{hive_log_maxfilesize}}MB\nlog4j.appender.DRFA.layout=org.apache.log4j.PatternLayout\n\n\n# Pattern format: Date LogLevel LoggerName LogMessage\n#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\n# Debugging Pattern format\nlog4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n\n\n\n#\n# console\n# Add \"console\" to rootlogger above if you want to use this\n#\n\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\nlog4j.appender.console.target=System.err\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\nlog4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n\nlog4j.appender.console.encoding=UTF-8\n\n#custom logging levels\n#log4j.logger.xxx=DEBUG\n\n#\n# Event Counter Appender\n# Sends counts of logging messages at different severity levels to Hadoop Metrics.\n#\nlog4j.appender.EventCounter=org.apache.hadoop.hive.shims.HiveEventCounter\n\n\nlog4j.category.DataNucleus=ERROR,DRFA\nlog4j.category.Datastore=ERROR,DRFA\nlog4j.category.Datastore.Schema=ERROR,DRFA\nlog4j.category.JPOX.Datastore=ERROR,DRFA\nlog4j.category.JPOX.Plugin=ERROR,DRFA\nlog4j.category.JPOX.MetaData=ERROR,DRFA\nlog4j.category.JPOX.Query=ERROR,DRFA\nlog4j.category.JPOX.General=ERROR,DRFA\nlog4j.category.JPOX.Enhancer=ERROR,DRFA\n\n\n# Silence useless ZK logs\nlog4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN,DRFA\nlog4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN,DRFA"
  596. }
  597. }
  598. },
  599. {
  600. "slider-client" : {
  601. "properties_attributes" : { },
  602. "properties" : { }
  603. }
  604. },
  605. {
  606. "activity-env" : {
  607. "properties_attributes" : { },
  608. "properties" : {
  609. "analyzer_jvm_heap" : "8192",
  610. "analyzer_jvm_opts" : "-Xms128m",
  611. "activity-env-content" : "#!/bin/bash\n\n# Copyright (c) 2011-2017, Hortonworks Inc. All rights reserved.\n# Except as expressly permitted in a written agreement between you\n# or your company and Hortonworks, Inc, any use, reproduction,\n# modification, redistribution, sharing, lending or other exploitation\n# of all or any part of the contents of this file is strictly prohibited.\n\n# Enable verbose shell execution\n#set -xv\n\n## Set HOME for various components\nexport HADOOP_HOME=/usr/hdp/current/hadoop-client\nexport HDFS_HOME=/usr/hdp/current/hadoop-hdfs-client\nexport MAPREDUCE_HOME=/usr/hdp/current/hadoop-mapreduce-client\nexport YARN_HOME=/usr/hdp/current/hadoop-yarn-client\nexport HIVE_HOME=/usr/hdp/current/hive-client\nexport HCAT_HOME=/usr/hdp/current/hive-webhcat\nexport TEZ_HOME=/usr/hdp/current/tez-client\nexport HBASE_HOME=/usr/hdp/current/hbase-client\nexport PHOENIX_HOME=/usr/hdp/current/phoenix-client\nexport ACTIVITY_ANALYZER_HOME=/usr/hdp/share/hst/activity-analyzer\nexport AMS_COLLECTOR_HOME=/usr/lib/ambari-metrics-collector\nexport JAVA_HOME={{java_home}}\n\n## Set conf dir for various components\nexport HADOOP_CONF_DIR=/etc/hadoop/conf/\nexport HIVE_CONF_DIR=/etc/hive/conf/\nexport HBASE_CONF_DIR=/etc/hbase/conf/\nexport TEZ_CONF_DIR=/etc/tez/conf/\nexport ACTIVITY_ANALYZER_CONF_DIR=/etc/smartsense-activity/conf/\nexport AMS_HBASE_CONF=/etc/ams-hbase/conf\n\nexport DEBUG_ENABLED=false\n\n## Set JVM related configs\nexport ANALYZER_JAVA_OPTS=\"{{analyzer_jvm_opts}} -Xmx{{analyzer_jvm_heap}}m\""
  612. }
  613. }
  614. },
  615. {
  616. "activity-zeppelin-env" : {
  617. "properties_attributes" : { },
  618. "properties" : {
  619. "activity-zeppelin-env-content" : "#!/bin/bash\n\n# Copyright (c) 2011-2017, Hortonworks Inc. All rights reserved.\n# Except as expressly permitted in a written agreement between you\n# or your company and Hortonworks, Inc, any use, reproduction,\n# modification, redistribution, sharing, lending or other exploitation\n# of all or any part of the contents of this file is strictly prohibited.\n\n\nexport JAVA_HOME={{java_home}}\n# Additional jvm options. for example, export ZEPPELIN_JAVA_OPTS=\"-Dspark.executor.memory=8g -Dspark.cores.max=16\"\nexport ZEPPELIN_JAVA_OPTS=\"-Dhdp.version={{hdp_version}} -Dlog.file.name=activity-explorer.log -DSmartSenseActivityExplorer\" \t\t\n# export ZEPPELIN_INTP_MEM \t\t# zeppelin interpreter process jvm mem options. Default = ZEPPELIN_MEM\n# export ZEPPELIN_INTP_JAVA_OPTS \t\t# zeppelin interpreter process jvm options. Default = ZEPPELIN_JAVA_OPTS\n\nexport ZEPPELIN_MEM=\" -Xms256m -Xmx1024m -XX:MaxPermSize=256m\"\nexport ZEPPELIN_LOG_DIR={{activity_log_dir}}\nexport ZEPPELIN_PID_DIR=/var/run/smartsense-activity-explorer\nexport ZEPPELIN_WAR_TEMPDIR=/var/lib/smartsense/activity-explorer/webapp\nexport ZEPPELIN_NOTEBOOK_DIR=/var/lib/smartsense/activity-explorer/notebook\nexport ZEPPELIN_CLASSPATH=\"/etc/ams-hbase/conf:${ZEPPELIN_CLASSPATH}\"\nexport CLASSPATH=${ZEPPELIN_CLASSPATH}"
  620. }
  621. }
  622. },
  623. {
  624. "yarn-logsearch-conf" : {
  625. "properties_attributes" : { },
  626. "properties" : {
  627. "component_mappings" : "RESOURCEMANAGER:yarn_resourcemanager,yarn_historyserver,yarn_jobsummary;NODEMANAGER:yarn_nodemanager;APP_TIMELINE_SERVER:yarn_timelineserver",
  628. "content" : "\n{\n \"input\":[\n {\n \"type\":\"yarn_nodemanager\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/yarn-env/yarn_user', 'yarn')}}/yarn-{{default('configurations/yarn-env/yarn_user', 'yarn')}}-nodemanager-*.log\"\n },\n {\n \"type\":\"yarn_resourcemanager\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/yarn-env/yarn_user', 'yarn')}}/yarn-{{default('configurations/yarn-env/yarn_user', 'yarn')}}-resourcemanager-*.log\"\n },\n {\n \"type\":\"yarn_timelineserver\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/yarn-env/yarn_user', 'yarn')}}/yarn-{{default('configurations/yarn-env/yarn_user', 'yarn')}}-timelineserver-*.log\"\n },\n {\n \"type\":\"yarn_historyserver\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/yarn-env/yarn_user', 'yarn')}}/yarn-{{default('configurations/yarn-env/yarn_user', 'yarn')}}-historyserver-*.log\"\n },\n {\n \"type\":\"yarn_jobsummary\",\n \"rowtype\":\"service\",\n \"path\":\"{{default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/yarn-env/yarn_user', 'yarn')}}/hadoop-mapreduce.jobsummary.log\"\n }\n ],\n \"filter\":[\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"yarn_historyserver\",\n \"yarn_jobsummary\",\n \"yarn_nodemanager\",\n \"yarn_resourcemanager\",\n \"yarn_timelineserver\"\n ]\n }\n },\n \"log4j_format\":\"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\\\(%{INT:line_number}\\\\)\\\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"target_date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n }\n }\n }\n ]\n}",
  629. "service_name" : "YARN"
  630. }
  631. }
  632. },
  633. {
  634. "tez-env" : {
  635. "properties_attributes" : { },
  636. "properties" : {
  637. "heap_dump_location" : "/tmp",
  638. "content" : "\n# Tez specific configuration\nexport TEZ_CONF_DIR={{config_dir}}\n\n# Set HADOOP_HOME to point to a specific hadoop install directory\nexport HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}\n\n# The java implementation to use.\nexport JAVA_HOME={{java64_home}}",
  639. "enable_heap_dump" : "false",
  640. "tez_user" : "tez"
  641. }
  642. }
  643. },
  644. {
  645. "activity-zeppelin-shiro" : {
  646. "properties_attributes" : { },
  647. "properties" : {
  648. "main.securityManager.sessionManager" : "$sessionManager",
  649. "main.securityManager.sessionManager.globalSessionTimeout" : "86400000",
  650. "main.sessionManager" : "org.apache.shiro.web.session.mgt.DefaultWebSessionManager"
  651. }
  652. }
  653. },
  654. {
  655. "ams-hbase-security-site" : {
  656. "properties_attributes" : { },
  657. "properties" : {
  658. "hbase.coprocessor.master.classes" : "",
  659. "hbase.myclient.keytab" : "",
  660. "hadoop.security.authentication" : "",
  661. "ams.zookeeper.keytab" : "",
  662. "hbase.zookeeper.property.jaasLoginRenew" : "",
  663. "hbase.regionserver.keytab.file" : "",
  664. "hbase.zookeeper.property.authProvider.1" : "",
  665. "hbase.zookeeper.property.kerberos.removeHostFromPrincipal" : "",
  666. "hbase.coprocessor.region.classes" : "",
  667. "hbase.security.authentication" : "",
  668. "hbase.master.keytab.file" : "",
  669. "hbase.security.authorization" : "",
  670. "hbase.zookeeper.property.kerberos.removeRealmFromPrincipal" : ""
  671. }
  672. }
  673. },
  674. {
  675. "ranger-yarn-plugin-properties" : {
  676. "properties_attributes" : { },
  677. "properties" : { }
  678. }
  679. },
  680. {
  681. "yarn-env" : {
  682. "properties_attributes" : { },
  683. "properties" : {
  684. "yarn_cgroups_enabled" : "false",
  685. "yarn_user_nproc_limit" : "65536",
  686. "yarn_log_dir_prefix" : "/var/log/hadoop-yarn",
  687. "apptimelineserver_heapsize" : "8072",
  688. "yarn_user_nofile_limit" : "32768",
  689. "content" : "\n export HADOOP_YARN_HOME={{hadoop_yarn_home}}\n export YARN_LOG_DIR={{yarn_log_dir_prefix}}/$USER\n export YARN_PID_DIR={{yarn_pid_dir_prefix}}/$USER\n export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}\n export JAVA_HOME={{java64_home}}\n export JAVA_LIBRARY_PATH=\"${JAVA_LIBRARY_PATH}:{{hadoop_java_io_tmpdir}}\"\n\n # We need to add the EWMA appender for the yarn daemons only;\n # however, YARN_ROOT_LOGGER is shared by the yarn client and the\n # daemons. This is restrict the EWMA appender to daemons only.\n INVOKER=\"${0##*/}\"\n if [ \"$INVOKER\" == \"yarn-daemon.sh\" ]; then\n export YARN_ROOT_LOGGER=${YARN_ROOT_LOGGER:-INFO,EWMA,RFA}\n fi\n\n # User for YARN daemons\n export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn}\n\n # resolve links - $0 may be a softlink\n export YARN_CONF_DIR=\"${YARN_CONF_DIR:-$HADOOP_YARN_HOME/conf}\"\n\n # some Java parameters\n # export JAVA_HOME=/home/y/libexec/jdk1.6.0/\n if [ \"$JAVA_HOME\" != \"\" ]; then\n #echo \"run java in $JAVA_HOME\"\n JAVA_HOME=$JAVA_HOME\n fi\n\n if [ \"$JAVA_HOME\" = \"\" ]; then\n echo \"Error: JAVA_HOME is not set.\"\n exit 1\n fi\n\n JAVA=$JAVA_HOME/bin/java\n JAVA_HEAP_MAX=-Xmx1000m\n\n # For setting YARN specific HEAP sizes please use this\n # Parameter and set appropriately\n YARN_HEAPSIZE={{yarn_heapsize}}\n\n # check envvars which might override default args\n if [ \"$YARN_HEAPSIZE\" != \"\" ]; then\n JAVA_HEAP_MAX=\"-Xmx\"\"$YARN_HEAPSIZE\"\"m\"\n fi\n\n # Resource Manager specific parameters\n\n # Specify the max Heapsize for the ResourceManager using a numerical value\n # in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set\n # the value to 1000.\n # This value will be overridden by an Xmx setting specified in either YARN_OPTS\n # and/or YARN_RESOURCEMANAGER_OPTS.\n # If not specified, the default value will be picked from either YARN_HEAPMAX\n # or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.\n export YARN_RESOURCEMANAGER_HEAPSIZE={{resourcemanager_heapsize}}\n\n # Specify the JVM options to be used when starting the ResourceManager.\n # These options will be appended to the options specified as YARN_OPTS\n # and therefore may override any similar flags set in YARN_OPTS\n #export YARN_RESOURCEMANAGER_OPTS=\n\n # Node Manager specific parameters\n\n # Specify the max Heapsize for the NodeManager using a numerical value\n # in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set\n # the value to 1000.\n # This value will be overridden by an Xmx setting specified in either YARN_OPTS\n # and/or YARN_NODEMANAGER_OPTS.\n # If not specified, the default value will be picked from either YARN_HEAPMAX\n # or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.\n export YARN_NODEMANAGER_HEAPSIZE={{nodemanager_heapsize}}\n\n # Specify the max Heapsize for the timeline server using a numerical value\n # in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set\n # the value to 1024.\n # This value will be overridden by an Xmx setting specified in either YARN_OPTS\n # and/or YARN_TIMELINESERVER_OPTS.\n # If not specified, the default value will be picked from either YARN_HEAPMAX\n # or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two.\n export YARN_TIMELINESERVER_HEAPSIZE={{apptimelineserver_heapsize}}\n\n # Specify the JVM options to be used when starting the NodeManager.\n # These options will be appended to the options specified as YARN_OPTS\n # and therefore may override any similar flags set in YARN_OPTS\n #export YARN_NODEMANAGER_OPTS=\n\n # so that filenames w/ spaces are handled correctly in loops below\n IFS=\n\n\n # default log directory and file\n if [ \"$YARN_LOG_DIR\" = \"\" ]; then\n YARN_LOG_DIR=\"$HADOOP_YARN_HOME/logs\"\n fi\n if [ \"$YARN_LOGFILE\" = \"\" ]; then\n YARN_LOGFILE='yarn.log'\n fi\n\n # default policy file for service-level authorization\n if [ \"$YARN_POLICYFILE\" = \"\" ]; then\n YARN_POLICYFILE=\"hadoop-policy.xml\"\n fi\n\n # restore ordinary behaviour\n unset IFS\n\n\n YARN_OPTS=\"$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR\"\n YARN_OPTS=\"$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR\"\n YARN_OPTS=\"$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE\"\n YARN_OPTS=\"$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE\"\n YARN_OPTS=\"$YARN_OPTS -Dyarn.home.dir=$YARN_COMMON_HOME\"\n YARN_OPTS=\"$YARN_OPTS -Dyarn.id.str=$YARN_IDENT_STRING\"\n YARN_OPTS=\"$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-INFO,console}\"\n YARN_OPTS=\"$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}\"\n export YARN_NODEMANAGER_OPTS=\"$YARN_NODEMANAGER_OPTS -Dnm.audit.logger=INFO,NMAUDIT\"\n export YARN_RESOURCEMANAGER_OPTS=\"$YARN_RESOURCEMANAGER_OPTS -Drm.audit.logger=INFO,RMAUDIT\"\n if [ \"x$JAVA_LIBRARY_PATH\" != \"x\" ]; then\n YARN_OPTS=\"$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH\"\n fi\n YARN_OPTS=\"$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE\"\n YARN_OPTS=\"$YARN_OPTS -Djava.io.tmpdir={{hadoop_java_io_tmpdir}}\"\n {% if rm_security_opts is defined %}\n YARN_OPTS=\"{{rm_security_opts}} $YARN_OPTS\"\n {% endif %}",
  690. "is_supported_yarn_ranger" : "true",
  691. "service_check.queue.name" : "default",
  692. "resourcemanager_heapsize" : "1024",
  693. "yarn_pid_dir_prefix" : "/var/run/hadoop-yarn",
  694. "yarn_user" : "yarn",
  695. "min_user_id" : "1000",
  696. "yarn_heapsize" : "1024",
  697. "nodemanager_heapsize" : "1024"
  698. }
  699. }
  700. },
  701. {
  702. "hst-server-conf" : {
  703. "properties_attributes" : { },
  704. "properties" : {
  705. "customer.account.name" : "unspecified",
  706. "server.ssl_enabled" : "false",
  707. "agent.request.processing.timeout" : "7200",
  708. "client.threadpool.size.max" : "40",
  709. "server.max.heap" : "2048",
  710. "server.storage.dir" : "/var/lib/smartsense/hst-server/data",
  711. "customer.enable.flex.subscription" : "false",
  712. "gateway.registration.port" : "9450",
  713. "gateway.port" : "9451",
  714. "customer.flex.subscription.id" : "",
  715. "customer.notification.email" : "unspecified",
  716. "agent.request.syncup.interval" : "180",
  717. "gateway.host" : "embedded",
  718. "server.port" : "9000",
  719. "server.tmp.dir" : "/var/lib/smartsense/hst-server/tmp",
  720. "server.url" : "http://sandbox-hdp.hortonworks.com:9000",
  721. "customer.smartsense.id" : "unspecified"
  722. }
  723. }
  724. },
  725. {
  726. "ranger-hdfs-audit" : {
  727. "properties_attributes" : { },
  728. "properties" : { }
  729. }
  730. },
  731. {
  732. "ranger-hdfs-plugin-properties" : {
  733. "properties_attributes" : { },
  734. "properties" : { }
  735. }
  736. },
  737. {
  738. "beeline-log4j2" : {
  739. "properties_attributes" : { },
  740. "properties" : {
  741. "content" : "\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nstatus = INFO\nname = BeelineLog4j2\npackages = org.apache.hadoop.hive.ql.log\n\n# list of properties\nproperty.hive.log.level = {{hive_log_level}}\nproperty.hive.root.logger = console\n\n# list of all appenders\nappenders = console\n\n# console appender\nappender.console.type = Console\nappender.console.name = console\nappender.console.target = SYSTEM_ERR\nappender.console.layout.type = PatternLayout\nappender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n\n\n# list of all loggers\nloggers = HiveConnection\n\n# HiveConnection logs useful info for dynamic service discovery\nlogger.HiveConnection.name = org.apache.hive.jdbc.HiveConnection\nlogger.HiveConnection.level = INFO\n\n# root logger\nrootLogger.level = ${sys:hive.log.level}\nrootLogger.appenderRefs = root\nrootLogger.appenderRef.root.ref = ${sys:hive.root.logger}"
  742. }
  743. }
  744. },
  745. {
  746. "ams-env" : {
  747. "properties_attributes" : { },
  748. "properties" : {
  749. "timeline.metrics.skip.disk.metrics.patterns" : "true",
  750. "ambari_metrics_user" : "ams",
  751. "metrics_monitor_log_dir" : "/var/log/ambari-metrics-monitor",
  752. "metrics_collector_heapsize" : "512",
  753. "failover_strategy_blacklisted_interval" : "300",
  754. "metrics_collector_pid_dir" : "/var/run/ambari-metrics-collector",
  755. "metrics_collector_log_dir" : "/var/log/ambari-metrics-collector",
  756. "metrics_monitor_pid_dir" : "/var/run/ambari-metrics-monitor",
  757. "ams_classpath_additional" : "",
  758. "content" : "\n# Set environment variables here.\n\n# AMS instance name\nexport AMS_INSTANCE_NAME={{hostname}}\n\n# The java implementation to use. Java 1.6 required.\nexport JAVA_HOME={{java64_home}}\n\n# Collector Log directory for log4j\nexport AMS_COLLECTOR_LOG_DIR={{ams_collector_log_dir}}\n\n# Monitor Log directory for outfile\nexport AMS_MONITOR_LOG_DIR={{ams_monitor_log_dir}}\n\n# Collector pid directory\nexport AMS_COLLECTOR_PID_DIR={{ams_collector_pid_dir}}\n\n# Monitor pid directory\nexport AMS_MONITOR_PID_DIR={{ams_monitor_pid_dir}}\n\n# AMS HBase pid directory\nexport AMS_HBASE_PID_DIR={{hbase_pid_dir}}\n\n# AMS Collector heapsize\nexport AMS_COLLECTOR_HEAPSIZE={{metrics_collector_heapsize}}\n\n# HBase Tables Initialization check enabled\nexport AMS_HBASE_INIT_CHECK_ENABLED={{ams_hbase_init_check_enabled}}\n\n# AMS Collector options\nexport AMS_COLLECTOR_OPTS=\"-Djava.library.path=/usr/lib/ams-hbase/lib/hadoop-native\"\n{% if security_enabled %}\nexport AMS_COLLECTOR_OPTS=\"$AMS_COLLECTOR_OPTS -Djava.security.auth.login.config={{ams_collector_jaas_config_file}}\"\n{% endif %}\n\n# AMS Collector GC options\nexport AMS_COLLECTOR_GC_OPTS=\"-XX:+UseConcMarkSweepGC -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:{{ams_collector_log_dir}}/collector-gc.log-`date +'%Y%m%d%H%M'`\"\nexport AMS_COLLECTOR_OPTS=\"$AMS_COLLECTOR_OPTS $AMS_COLLECTOR_GC_OPTS\"\n\n# Metrics collector host will be blacklisted for specified number of seconds if metric monitor failed to connect to it.\nexport AMS_FAILOVER_STRATEGY_BLACKLISTED_INTERVAL={{failover_strategy_blacklisted_interval}}\n\n# Extra Java CLASSPATH elements for Metrics Collector. Optional.\nexport COLLECTOR_ADDITIONAL_CLASSPATH={{ams_classpath_additional}}"
  759. }
  760. }
  761. },
  762. {
  763. "llap-cli-log4j2" : {
  764. "properties_attributes" : { },
  765. "properties" : {
  766. "llap_cli_log_maxfilesize" : "256",
  767. "content" : "\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nstatus = WARN\nname = LlapCliLog4j2\npackages = org.apache.hadoop.hive.ql.log\n\n# list of properties\nproperty.hive.log.level = WARN\nproperty.hive.root.logger = console\nproperty.hive.log.dir = ${sys:java.io.tmpdir}/${sys:user.name}\nproperty.hive.log.file = llap-cli.log\nproperty.hive.llapstatus.consolelogger.level = INFO\n\n# list of all appenders\nappenders = console, DRFA, llapstatusconsole\n\n# console appender\nappender.console.type = Console\nappender.console.name = console\nappender.console.target = SYSTEM_ERR\nappender.console.layout.type = PatternLayout\nappender.console.layout.pattern = %p %c{2}: %m%n\n\n# llapstatusconsole appender\nappender.llapstatusconsole.type = Console\nappender.llapstatusconsole.name = llapstatusconsole\nappender.llapstatusconsole.target = SYSTEM_OUT\nappender.llapstatusconsole.layout.type = PatternLayout\nappender.llapstatusconsole.layout.pattern = %m%n\n\n# daily rolling file appender\nappender.DRFA.type = RollingRandomAccessFile\nappender.DRFA.name = DRFA\nappender.DRFA.fileName = ${sys:hive.log.dir}/${sys:hive.log.file}\n# Use %pid in the filePattern to append process-id@host-name to the filename if you want separate log files for different CLI session\nappender.DRFA.filePattern = ${sys:hive.log.dir}/${sys:hive.log.file}.%d{yyyy-MM-dd}_%i\nappender.DRFA.layout.type = PatternLayout\nappender.DRFA.layout.pattern = %d{ISO8601} %5p [%t] %c{2}: %m%n\nappender.DRFA.policies.type = Policies\nappender.DRFA.policies.time.type = TimeBasedTriggeringPolicy\nappender.DRFA.policies.time.interval = 1\nappender.DRFA.policies.time.modulate = true\nappender.DRFA.strategy.type = DefaultRolloverStrategy\nappender.DRFA.strategy.max = {{llap_cli_log_maxbackupindex}}\nappender.DRFA.policies.fsize.type = SizeBasedTriggeringPolicy\nappender.DRFA.policies.fsize.size = {{llap_cli_log_maxfilesize}}MB\n\n# list of all loggers\nloggers = ZooKeeper, DataNucleus, Datastore, JPOX, HadoopConf, LlapStatusServiceDriverConsole\n\nlogger.ZooKeeper.name = org.apache.zookeeper\nlogger.ZooKeeper.level = WARN\n\nlogger.DataNucleus.name = DataNucleus\nlogger.DataNucleus.level = ERROR\n\nlogger.Datastore.name = Datastore\nlogger.Datastore.level = ERROR\n\nlogger.JPOX.name = JPOX\nlogger.JPOX.level = ERROR\n\nlogger.HadoopConf.name = org.apache.hadoop.conf.Configuration\nlogger.HadoopConf.level = ERROR\n\nlogger.LlapStatusServiceDriverConsole.name = LlapStatusServiceDriverConsole\nlogger.LlapStatusServiceDriverConsole.additivity = false\nlogger.LlapStatusServiceDriverConsole.level = ${sys:hive.llapstatus.consolelogger.level}\n\n\n# root logger\nrootLogger.level = ${sys:hive.log.level}\nrootLogger.appenderRefs = root, DRFA\nrootLogger.appenderRef.root.ref = ${sys:hive.root.logger}\nrootLogger.appenderRef.DRFA.ref = DRFA\nlogger.LlapStatusServiceDriverConsole.appenderRefs = llapstatusconsole, DRFA\nlogger.LlapStatusServiceDriverConsole.appenderRef.llapstatusconsole.ref = llapstatusconsole\nlogger.LlapStatusServiceDriverConsole.appenderRef.DRFA.ref = DRFA",
  768. "llap_cli_log_maxbackupindex" : "30"
  769. }
  770. }
  771. },
  772. {
  773. "llap-daemon-log4j" : {
  774. "properties_attributes" : { },
  775. "properties" : {
  776. "hive_llap_log_maxfilesize" : "256",
  777. "content" : "\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\n# This is the log4j2 properties file used by llap-daemons. There's several loggers defined, which\n# can be selected while configuring LLAP.\n# Based on the one selected - UI links etc need to be manipulated in the system.\n# Note: Some names and logic is common to this file and llap LogHelpers. Make sure to change that\n# as well, if changing this file.\n\nstatus = INFO\nname = LlapDaemonLog4j2\npackages = org.apache.hadoop.hive.ql.log\n\n# list of properties\nproperty.llap.daemon.log.level = {{hive_log_level}}\nproperty.llap.daemon.root.logger = console\nproperty.llap.daemon.log.dir = .\nproperty.llap.daemon.log.file = llapdaemon.log\nproperty.llap.daemon.historylog.file = llapdaemon_history.log\nproperty.llap.daemon.log.maxfilesize = {{hive_llap_log_maxfilesize}}MB\nproperty.llap.daemon.log.maxbackupindex = {{hive_llap_log_maxbackupindex}}\n\n# list of all appenders\nappenders = console, RFA, HISTORYAPPENDER, query-routing\n\n# console appender\nappender.console.type = Console\nappender.console.name = console\nappender.console.target = SYSTEM_ERR\nappender.console.layout.type = PatternLayout\nappender.console.layout.pattern = %d{ISO8601} %5p [%t (%X{fragmentId})] %c{2}: %m%n\n\n# rolling file appender\nappender.RFA.type = RollingRandomAccessFile\nappender.RFA.name = RFA\nappender.RFA.fileName = ${sys:llap.daemon.log.dir}/${sys:llap.daemon.log.file}\nappender.RFA.filePattern = ${sys:llap.daemon.log.dir}/${sys:llap.daemon.log.file}_%d{yyyy-MM-dd-HH}_%i.done\nappender.RFA.layout.type = PatternLayout\nappender.RFA.layout.pattern = %d{ISO8601} %-5p [%t (%X{fragmentId})] %c: %m%n\nappender.RFA.policies.type = Policies\nappender.RFA.policies.time.type = TimeBasedTriggeringPolicy\nappender.RFA.policies.time.interval = 1\nappender.RFA.policies.time.modulate = true\nappender.RFA.policies.size.type = SizeBasedTriggeringPolicy\nappender.RFA.policies.size.size = ${sys:llap.daemon.log.maxfilesize}\nappender.RFA.strategy.type = DefaultRolloverStrategy\nappender.RFA.strategy.max = ${sys:llap.daemon.log.maxbackupindex}\n\n# history file appender\nappender.HISTORYAPPENDER.type = RollingRandomAccessFile\nappender.HISTORYAPPENDER.name = HISTORYAPPENDER\nappender.HISTORYAPPENDER.fileName = ${sys:llap.daemon.log.dir}/${sys:llap.daemon.historylog.file}\nappender.HISTORYAPPENDER.filePattern = ${sys:llap.daemon.log.dir}/${sys:llap.daemon.historylog.file}_%d{yyyy-MM-dd}_%i.done\nappender.HISTORYAPPENDER.layout.type = PatternLayout\nappender.HISTORYAPPENDER.layout.pattern = %m%n\nappender.HISTORYAPPENDER.policies.type = Policies\nappender.HISTORYAPPENDER.policies.size.type = SizeBasedTriggeringPolicy\nappender.HISTORYAPPENDER.policies.size.size = ${sys:llap.daemon.log.maxfilesize}\nappender.HISTORYAPPENDER.policies.time.type = TimeBasedTriggeringPolicy\nappender.HISTORYAPPENDER.policies.time.interval = 1\nappender.HISTORYAPPENDER.policies.time.modulate = true\nappender.HISTORYAPPENDER.strategy.type = DefaultRolloverStrategy\nappender.HISTORYAPPENDER.strategy.max = ${sys:llap.daemon.log.maxbackupindex}\n\n# queryId based routing file appender\nappender.query-routing.type = Routing\nappender.query-routing.name = query-routing\nappender.query-routing.routes.type = Routes\nappender.query-routing.routes.pattern = $${ctx:queryId}\n#Purge polciy for query-based Routing Appender\nappender.query-routing.purgePolicy.type = LlapRoutingAppenderPurgePolicy\n# Note: Do not change this name without changing the corresponding entry in LlapConstants\nappender.query-routing.purgePolicy.name = llapLogPurgerQueryRouting\n# default route\nappender.query-routing.routes.route-default.type = Route\nappender.query-routing.routes.route-default.key = $${ctx:queryId}\nappender.query-routing.routes.route-default.ref = RFA\n# queryId based route\nappender.query-routing.routes.route-mdc.type = Route\nappender.query-routing.routes.route-mdc.file-mdc.type = LlapWrappedAppender\nappender.query-routing.routes.route-mdc.file-mdc.name = IrrelevantName-query-routing\nappender.query-routing.routes.route-mdc.file-mdc.app.type = RandomAccessFile\nappender.query-routing.routes.route-mdc.file-mdc.app.name = file-mdc\nappender.query-routing.routes.route-mdc.file-mdc.app.fileName = ${sys:llap.daemon.log.dir}/${ctx:queryId}-${ctx:dagId}.log\nappender.query-routing.routes.route-mdc.file-mdc.app.layout.type = PatternLayout\nappender.query-routing.routes.route-mdc.file-mdc.app.layout.pattern = %d{ISO8601} %5p [%t (%X{fragmentId})] %c{2}: %m%n\n\n# list of all loggers\nloggers = PerfLogger, EncodedReader, NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX, HistoryLogger, LlapIoImpl, LlapIoOrc, LlapIoCache, LlapIoLocking, TezSM, TezSS, TezHC\n\nlogger.TezSM.name = org.apache.tez.runtime.library.common.shuffle.impl.ShuffleManager.fetch\nlogger.TezSM.level = WARN\nlogger.TezSS.name = org.apache.tez.runtime.library.common.shuffle.orderedgrouped.ShuffleScheduler.fetch\nlogger.TezSS.level = WARN\nlogger.TezHC.name = org.apache.tez.http.HttpConnection.url\nlogger.TezHC.level = WARN\n\nlogger.PerfLogger.name = org.apache.hadoop.hive.ql.log.PerfLogger\nlogger.PerfLogger.level = DEBUG\n\nlogger.EncodedReader.name = org.apache.hadoop.hive.ql.io.orc.encoded.EncodedReaderImpl\nlogger.EncodedReader.level = INFO\n\nlogger.LlapIoImpl.name = LlapIoImpl\nlogger.LlapIoImpl.level = INFO\n\nlogger.LlapIoOrc.name = LlapIoOrc\nlogger.LlapIoOrc.level = WARN\n\nlogger.LlapIoCache.name = LlapIoCache\nlogger.LlapIoCache.level = WARN\n\nlogger.LlapIoLocking.name = LlapIoLocking\nlogger.LlapIoLocking.level = WARN\n\nlogger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn\nlogger.NIOServerCnxn.level = WARN\n\nlogger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO\nlogger.ClientCnxnSocketNIO.level = WARN\n\nlogger.DataNucleus.name = DataNucleus\nlogger.DataNucleus.level = ERROR\n\nlogger.Datastore.name = Datastore\nlogger.Datastore.level = ERROR\n\nlogger.JPOX.name = JPOX\nlogger.JPOX.level = ERROR\n\nlogger.HistoryLogger.name = org.apache.hadoop.hive.llap.daemon.HistoryLogger\nlogger.HistoryLogger.level = INFO\nlogger.HistoryLogger.additivity = false\nlogger.HistoryLogger.appenderRefs = HistoryAppender\nlogger.HistoryLogger.appenderRef.HistoryAppender.ref = HISTORYAPPENDER\n\n# root logger\nrootLogger.level = ${sys:llap.daemon.log.level}\nrootLogger.appenderRefs = root\nrootLogger.appenderRef.root.ref = ${sys:llap.daemon.root.logger}",
  778. "hive_llap_log_maxbackupindex" : "240"
  779. }
  780. }
  781. },
  782. {
  783. "hst-log4j" : {
  784. "properties_attributes" : { },
  785. "properties" : {
  786. "hst_max_backup_index" : "10",
  787. "hst_max_file_size" : "30",
  788. "hst-log4j-content" : "\n# Copyright (c) 2011-2017, Hortonworks Inc. All rights reserved.\n# Except as expressly permitted in a written agreement between you\n# or your company and Hortonworks, Inc, any use, reproduction,\n# modification, redistribution, sharing, lending or other exploitation\n# of all or any part of the contents of this file is strictly prohibited.\n\n# Define some default values that can be overridden by system properties\n# Root logger option\nlog4j.rootLogger=INFO,file\n\nlog4j.appender.file=org.apache.log4j.RollingFileAppender\nlog4j.appender.file.File={{hst_log_dir}}/${log.file.name}\nlog4j.appender.file.MaxFileSize={{hst_max_file_size}}MB\nlog4j.appender.file.MaxBackupIndex={{hst_max_backup_index}}\nlog4j.appender.file.layout=org.apache.log4j.PatternLayout\nlog4j.appender.file.layout.ConversionPattern=%d{ISO8601} %5p [%t] %c{1}:%L - %m%n\n\n# HST logger\nlog4j.logger.com.hortonworks=INFO\ncom.github.oxo42.stateless4j=WARN\nlog4j.logger.com.sun.jersey=WARN\nlog4j.logger.org.eclipse.jetty.server=INFO",
  789. "hst_log_dir" : "/var/log/hst"
  790. }
  791. }
  792. },
  793. {
  794. "capacity-scheduler" : {
  795. "properties_attributes" : { },
  796. "properties" : {
  797. "capacity-scheduler" : "null",
  798. "yarn.scheduler.capacity.root.accessible-node-labels" : "*",
  799. "yarn.scheduler.capacity.default.minimum-user-limit-percent" : "100",
  800. "yarn.scheduler.capacity.maximum-am-resource-percent" : "0.2",
  801. "yarn.scheduler.capacity.root.acl_administer_queue" : "*",
  802. "yarn.scheduler.capacity.root.default.acl_administer_jobs" : "*",
  803. "yarn.scheduler.capacity.resource-calculator" : "org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator",
  804. "yarn.scheduler.capacity.root.default.capacity" : "100",
  805. "yarn.scheduler.capacity.root.default.user-limit-factor" : "1",
  806. "yarn.scheduler.capacity.root.queues" : "default",
  807. "yarn.scheduler.capacity.root.capacity" : "100",
  808. "yarn.scheduler.capacity.root.default.acl_submit_applications" : "*",
  809. "yarn.scheduler.capacity.root.default.maximum-capacity" : "100",
  810. "yarn.scheduler.capacity.node-locality-delay" : "40",
  811. "yarn.scheduler.capacity.maximum-applications" : "10000",
  812. "yarn.scheduler.capacity.root.default.state" : "RUNNING"
  813. }
  814. }
  815. },
  816. {
  817. "webhcat-log4j" : {
  818. "properties_attributes" : { },
  819. "properties" : {
  820. "webhcat_log_maxfilesize" : "256",
  821. "webhcat_log_maxbackupindex" : "20",
  822. "content" : "\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied. See the License for the\n# specific language governing permissions and limitations\n# under the License.\n\n# Define some default values that can be overridden by system properties\nwebhcat.root.logger = {{hive_log_level}}, standard\nwebhcat.log.dir = .\nwebhcat.log.file = webhcat.log\n\nlog4j.rootLogger = ${webhcat.root.logger}\n\n# Logging Threshold\nlog4j.threshhold = DEBUG\n\nlog4j.appender.standard = org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.standard.File = ${webhcat.log.dir}/${webhcat.log.file}\nlog4j.appender.standard.MaxFileSize = {{webhcat_log_maxfilesize}}MB\nlog4j.appender.standard.MaxBackupIndex = {{webhcat_log_maxbackupindex}}\n\n# Rollver at midnight\nlog4j.appender.DRFA.DatePattern = .yyyy-MM-dd\n\nlog4j.appender.DRFA.layout = org.apache.log4j.PatternLayout\n\nlog4j.appender.standard.layout = org.apache.log4j.PatternLayout\nlog4j.appender.standard.layout.conversionPattern = %-5p | %d{DATE} | %c | %m%n\n\n# Class logging settings\nlog4j.logger.com.sun.jersey = DEBUG\nlog4j.logger.com.sun.jersey.spi.container.servlet.WebComponent = ERROR\nlog4j.logger.org.apache.hadoop = INFO\nlog4j.logger.org.apache.hadoop.conf = WARN\nlog4j.logger.org.apache.zookeeper = WARN\nlog4j.logger.org.eclipse.jetty = INFO"
  823. }
  824. }
  825. },
  826. {
  827. "hadoop-metrics2.properties" : {
  828. "properties_attributes" : { },
  829. "properties" : {
  830. "content" : "\n# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements. See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# syntax: [prefix].[source|sink|jmx].[instance].[options]\n# See package.html for org.apache.hadoop.metrics2 for details\n\n{% if has_ganglia_server %}\n*.period=60\n\n*.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31\n*.sink.ganglia.period=10\n\n# default for supportsparse is false\n*.sink.ganglia.supportsparse=true\n\n.sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both\n.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40\n\n# Hook up to the server\nnamenode.sink.ganglia.servers={{ganglia_server_host}}:8661\ndatanode.sink.ganglia.servers={{ganglia_server_host}}:8659\njobtracker.sink.ganglia.servers={{ganglia_server_host}}:8662\ntasktracker.sink.ganglia.servers={{ganglia_server_host}}:8658\nmaptask.sink.ganglia.servers={{ganglia_server_host}}:8660\nreducetask.sink.ganglia.servers={{ganglia_server_host}}:8660\nresourcemanager.sink.ganglia.servers={{ganglia_server_host}}:8664\nnodemanager.sink.ganglia.servers={{ganglia_server_host}}:8657\nhistoryserver.sink.ganglia.servers={{ganglia_server_host}}:8666\njournalnode.sink.ganglia.servers={{ganglia_server_host}}:8654\nnimbus.sink.ganglia.servers={{ganglia_server_host}}:8649\nsupervisor.sink.ganglia.servers={{ganglia_server_host}}:8650\n\nresourcemanager.sink.ganglia.tagsForPrefix.yarn=Queue\n\n{% endif %}\n\n{% if has_metric_collector %}\n\n*.period={{metrics_collection_period}}\n*.sink.timeline.plugin.urls=file:///usr/lib/ambari-metrics-hadoop-sink/ambari-metrics-hadoop-sink.jar\n*.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink\n*.sink.timeline.period={{metrics_collection_period}}\n*.sink.timeline.sendInterval={{metrics_report_interval}}000\n*.sink.timeline.slave.host.name={{hostname}}\n*.sink.timeline.zookeeper.quorum={{zookeeper_quorum}}\n*.sink.timeline.protocol={{metric_collector_protocol}}\n*.sink.timeline.port={{metric_collector_port}}\n\n# HTTPS properties\n*.sink.timeline.truststore.path = {{metric_truststore_path}}\n*.sink.timeline.truststore.type = {{metric_truststore_type}}\n*.sink.timeline.truststore.password = {{metric_truststore_password}}\n\ndatanode.sink.timeline.collector.hosts={{ams_collector_hosts}}\nnamenode.sink.timeline.collector.hosts={{ams_collector_hosts}}\nresourcemanager.sink.timeline.collector.hosts={{ams_collector_hosts}}\nnodemanager.sink.timeline.collector.hosts={{ams_collector_hosts}}\njobhistoryserver.sink.timeline.collector.hosts={{ams_collector_hosts}}\njournalnode.sink.timeline.collector.hosts={{ams_collector_hosts}}\nmaptask.sink.timeline.collector.hosts={{ams_collector_hosts}}\nreducetask.sink.timeline.collector.hosts={{ams_collector_hosts}}\napplicationhistoryserver.sink.timeline.collector.hosts={{ams_collector_hosts}}\n\nresourcemanager.sink.timeline.tagsForPrefix.yarn=Queue\n\n{% if is_nn_client_port_configured %}\n# Namenode rpc ports customization\nnamenode.sink.timeline.metric.rpc.client.port={{nn_rpc_client_port}}\n{% endif %}\n{% if is_nn_dn_port_configured %}\nnamenode.sink.timeline.metric.rpc.datanode.port={{nn_rpc_dn_port}}\n{% endif %}\n{% if is_nn_healthcheck_port_configured %}\nnamenode.sink.timeline.metric.rpc.healthcheck.port={{nn_rpc_healthcheck_port}}\n{% endif %}\n\n{% endif %}"
  831. }
  832. }
  833. },
  834. {
  835. "hive-interactive-env" : {
  836. "properties_attributes" : { },
  837. "properties" : {
  838. "enable_hive_interactive" : "false",
  839. "llap_heap_size" : "0",
  840. "llap_app_name" : "llap0",
  841. "num_retries_for_checking_llap_status" : "20",
  842. "hive_heapsize" : "512",
  843. "content" : "\nif [ \"$SERVICE\" = \"cli\" ]; then\n if [ -z \"$DEBUG\" ]; then\n export HADOOP_OPTS=\"$HADOOP_OPTS -XX:NewRatio=12 -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseParNewGC -XX:-UseGCOverheadLimit\"\n else\n export HADOOP_OPTS=\"$HADOOP_OPTS -XX:NewRatio=12 -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:-UseGCOverheadLimit\"\n fi\nfi\n\n# The heap size of the jvm stared by hive shell script can be controlled via:\n\nif [ \"$SERVICE\" = \"metastore\" ]; then\n export HADOOP_HEAPSIZE={{hive_metastore_heapsize}} # Setting for HiveMetastore\nelse\n export HADOOP_HEAPSIZE={{hive_interactive_heapsize}} # Setting for HiveServer2 and Client\nfi\n\nexport HADOOP_CLIENT_OPTS=\"$HADOOP_CLIENT_OPTS -Xmx${HADOOP_HEAPSIZE}m\"\nexport HADOOP_CLIENT_OPTS=\"$HADOOP_CLIENT_OPTS{{heap_dump_opts}}\"\n\n# Larger heap size may be required when running queries over large number of files or partitions.\n# By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be\n# appropriate for hive server (hwi etc).\n\n\n# Set HADOOP_HOME to point to a specific hadoop install directory\nHADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}\n\n# Hive Configuration Directory can be controlled by:\nexport HIVE_CONF_DIR={{hive_server_interactive_conf_dir}}\n\n# Add additional hcatalog jars\nif [ \"${HIVE_AUX_JARS_PATH}\" != \"\" ]; then\n export HIVE_AUX_JARS_PATH=${HIVE_AUX_JARS_PATH}\nelse\n export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-server2-hive2/lib/hive-hcatalog-core.jar\nfi\n\nexport METASTORE_PORT={{hive_metastore_port}}\n\n# Spark assembly contains a conflicting copy of HiveConf from hive-1.2\nexport HIVE_SKIP_SPARK_ASSEMBLY=true",
  844. "llap_log_level" : "INFO",
  845. "num_llap_nodes_for_llap_daemons" : "1",
  846. "num_llap_nodes" : "1",
  847. "llap_headroom_space" : "12288",
  848. "llap_extra_slider_opts" : "",
  849. "slider_am_container_mb" : "341",
  850. "hive_aux_jars" : "",
  851. "llap_java_opts" : "-XX:+AlwaysPreTouch {% if java_version > 7 %}-Xss512k -XX:+UseG1GC -XX:TLABSize=8m -XX:+ResizeTLAB -XX:+UseNUMA -XX:+AggressiveOpts -XX:InitiatingHeapOccupancyPercent=40 -XX:G1ReservePercent=20 -XX:MaxGCPauseMillis=200{% else %}-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC{% endif %}{{heap_dump_opts}}"
  852. }
  853. }
  854. },
  855. {
  856. "hiveserver2-interactive-site" : {
  857. "properties_attributes" : { },
  858. "properties" : {
  859. "hive.async.log.enabled" : "false",
  860. "hive.service.metrics.hadoop2.component" : "hiveserver2",
  861. "hive.metastore.metrics.enabled" : "true",
  862. "hive.service.metrics.reporter" : "HADOOP2"
  863. }
  864. }
  865. },
  866. {
  867. "ams-site" : {
  868. "properties_attributes" : { },
  869. "properties" : {
  870. "timeline.metrics.cluster.aggregator.hourly.interval" : "3600",
  871. "timeline.metrics.cluster.aggregator.minute.checkpointCutOffMultiplier" : "2",
  872. "timeline.metrics.cluster.aggregator.daily.checkpointCutOffMultiplier" : "2",
  873. "timeline.metrics.host.aggregator.hourly.interval" : "3600",
  874. "timeline.metrics.aggregators.skip.blockcache.enabled" : "false",
  875. "timeline.metrics.service.rpc.address" : "0.0.0.0:60200",
  876. "timeline.metrics.service.operation.mode" : "embedded",
  877. "timeline.metrics.aggregator.checkpoint.dir" : "/var/lib/ambari-metrics-collector/checkpoint",
  878. "failover.strategy" : "round-robin",
  879. "timeline.metrics.cluster.aggregator.second.checkpointCutOffMultiplier" : "2",
  880. "timeline.metrics.service.http.policy" : "HTTP_ONLY",
  881. "timeline.metrics.downsampler.topn.value" : "10",
  882. "timeline.metrics.host.aggregator.minute.checkpointCutOffMultiplier" : "2",
  883. "timeline.metrics.service.watcher.timeout" : "30",
  884. "timeline.metrics.service.checkpointDelay" : "60",
  885. "timeline.metrics.cluster.aggregator.second.interval" : "120",
  886. "timeline.metrics.service.webapp.address" : "0.0.0.0:6188",
  887. "timeline.metrics.host.aggregator.daily.ttl" : "31536000",
  888. "timeline.metrics.service.watcher.delay" : "30",
  889. "timeline.metrics.service.watcher.disabled" : "false",
  890. "timeline.metrics.hbase.init.check.enabled" : "true",
  891. "timeline.metrics.host.aggregator.hourly.disabled" : "false",
  892. "timeline.metrics.service.cluster.aggregator.appIds" : "datanode,nodemanager,hbase",
  893. "timeline.metrics.cluster.aggregator.hourly.checkpointCutOffMultiplier" : "2",
  894. "timeline.metrics.host.aggregator.daily.checkpointCutOffMultiplier" : "2",
  895. "timeline.metrics.service.resultset.fetchSize" : "2000",
  896. "timeline.metrics.cluster.aggregator.hourly.ttl" : "31536000",
  897. "cluster.zookeeper.quorum" : "{{cluster_zookeeper_quorum_hosts}}",
  898. "timeline.metrics.downsampler.topn.function" : "max",
  899. "timeline.metrics.host.aggregator.ttl" : "86400",
  900. "phoenix.spool.directory" : "/tmp",
  901. "timeline.metrics.host.aggregate.splitpoints" : "load_fifteen",
  902. "timeline.metrics.service.handler.thread.count" : "20",
  903. "timeline.metrics.cache.size" : "100",
  904. "timeline.metrics.cluster.aggregator.minute.interval" : "300",
  905. "timeline.metrics.cluster.aggregator.minute.ttl" : "2592000",
  906. "timeline.metrics.host.aggregator.minute.interval" : "300",
  907. "timeline.metrics.cluster.aggregator.interpolation.enabled" : "true",
  908. "timeline.metrics.cache.commit.interval" : "10",
  909. "timeline.metrics.host.aggregator.minute.disabled" : "false",
  910. "timeline.metrics.service.metadata.filters" : "ContainerResource",
  911. "timeline.metrics.cache.enabled" : "true",
  912. "timeline.metrics.cluster.aggregate.splitpoints" : "load_fifteen",
  913. "timeline.metrics.cluster.aggregator.minute.disabled" : "false",
  914. "timeline.metrics.service.use.groupBy.aggregators" : "true",
  915. "phoenix.query.maxGlobalMemoryPercentage" : "25",
  916. "timeline.metrics.service.default.result.limit" : "15840",
  917. "timeline.metrics.hbase.compression.scheme" : "SNAPPY",
  918. "timeline.metrics.cluster.aggregator.daily.ttl" : "63072000",
  919. "cluster.zookeeper.property.clientPort" : "{{cluster_zookeeper_clientPort}}",
  920. "timeline.metrics.sink.report.interval" : "60",
  921. "timeline.metrics.cluster.aggregator.second.timeslice.interval" : "30",
  922. "timeline.metrics.cluster.aggregation.sql.filters" : "sdisk\\_%,boottime",
  923. "timeline.metrics.downsampler.topn.metric.patterns" : "dfs.NNTopUserOpCounts.windowMs=60000.op=__%.user=%,dfs.NNTopUserOpCounts.windowMs=300000.op=__%.user=%,dfs.NNTopUserOpCounts.windowMs=1500000.op=__%.user=%",
  924. "timeline.metrics.host.aggregator.hourly.ttl" : "2592000",
  925. "timeline.metrics.cluster.aggregator.daily.interval" : "86400",
  926. "timeline.metrics.host.aggregator.daily.disabled" : "false",
  927. "timeline.metrics.cluster.aggregator.daily.disabled" : "false",
  928. "timeline.metrics.cluster.aggregator.hourly.disabled" : "false",
  929. "timeline.metrics.service.watcher.initial.delay" : "600",
  930. "timeline.metrics.host.aggregator.minute.ttl" : "604800",
  931. "timeline.metrics.hbase.data.block.encoding" : "FAST_DIFF",
  932. "timeline.metrics.cluster.aggregator.second.disabled" : "false",
  933. "timeline.metrics.sink.collection.period" : "10",
  934. "timeline.metrics.host.aggregator.hourly.checkpointCutOffMultiplier" : "2",
  935. "timeline.metrics.daily.aggregator.minute.interval" : "86400",
  936. "timeline.metrics.cluster.aggregator.second.ttl" : "259200"
  937. }
  938. }
  939. },
  940. {
  941. "mapred-site" : {
  942. "properties_attributes" : { },
  943. "properties" : {
  944. "mapreduce.map.speculative" : "false",
  945. "mapreduce.jobhistory.recovery.store.class" : "org.apache.hadoop.mapreduce.v2.hs.HistoryServerLeveldbStateStoreService",
  946. "mapreduce.job.counters.max" : "130",
  947. "mapreduce.reduce.log.level" : "INFO",
  948. "mapreduce.shuffle.port" : "13562",
  949. "yarn.app.mapreduce.am.admin-command-opts" : "-Dhdp.version=${hdp.version}",
  950. "mapreduce.job.emit-timeline-data" : "false",
  951. "mapreduce.jobhistory.recovery.enable" : "true",
  952. "mapreduce.map.log.level" : "INFO",
  953. "yarn.app.mapreduce.am.staging-dir" : "/user",
  954. "mapreduce.reduce.shuffle.merge.percent" : "0.66",
  955. "mapreduce.output.fileoutputformat.compress" : "false",
  956. "mapreduce.admin.map.child.java.opts" : "-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}",
  957. "mapreduce.job.reduce.slowstart.completedmaps" : "0.05",
  958. "mapreduce.jobhistory.http.policy" : "HTTP_ONLY",
  959. "mapreduce.job.queuename" : "default",
  960. "mapreduce.application.framework.path" : "/hdp/apps/${hdp.version}/mapreduce/mapreduce.tar.gz#mr-framework",
  961. "mapreduce.application.classpath" : "$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:$PWD/mr-framework/hadoop/share/hadoop/tools/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure:/usr/hdp/current/ext/hadoop/*",
  962. "mapreduce.reduce.java.opts" : "-Xmx3276m",
  963. "mapreduce.map.output.compress" : "false",
  964. "mapreduce.reduce.input.buffer.percent" : "0.0",
  965. "mapreduce.jobhistory.intermediate-done-dir" : "/mr-history/tmp",
  966. "yarn.app.mapreduce.am.log.level" : "INFO",
  967. "mapreduce.reduce.shuffle.fetch.retry.interval-ms" : "1000",
  968. "mapreduce.reduce.shuffle.input.buffer.percent" : "0.7",
  969. "mapreduce.reduce.speculative" : "false",
  970. "mapreduce.output.fileoutputformat.compress.type" : "BLOCK",
  971. "mapreduce.task.io.sort.mb" : "1146",
  972. "mapreduce.cluster.administrators" : " hadoop",
  973. "mapreduce.reduce.shuffle.fetch.retry.timeout-ms" : "30000",
  974. "mapreduce.jobhistory.address" : "%HOSTGROUP::host_group_1%:10020",
  975. "mapreduce.map.sort.spill.percent" : "0.7",
  976. "mapreduce.map.memory.mb" : "2048",
  977. "mapreduce.admin.reduce.child.java.opts" : "-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}",
  978. "mapreduce.jobhistory.bind-host" : "0.0.0.0",
  979. "mapreduce.task.timeout" : "300000",
  980. "mapreduce.admin.user.env" : "LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64",
  981. "mapreduce.am.max-attempts" : "2",
  982. "mapreduce.reduce.memory.mb" : "4096",
  983. "mapreduce.framework.name" : "yarn",
  984. "mapreduce.reduce.shuffle.parallelcopies" : "30",
  985. "mapreduce.jobhistory.done-dir" : "/mr-history/done",
  986. "mapreduce.jobhistory.webapp.address" : "%HOSTGROUP::host_group_1%:19888",
  987. "mapreduce.reduce.shuffle.fetch.retry.enabled" : "1",
  988. "mapreduce.task.io.sort.factor" : "100",
  989. "mapreduce.jobhistory.recovery.store.leveldb.path" : "/hadoop/mapreduce/jhs",
  990. "yarn.app.mapreduce.client.job.max-retries" : "60",
  991. "yarn.app.mapreduce.am.command-opts" : "-Xmx1638m -Dhdp.version=${hdp.version}",
  992. "mapreduce.map.java.opts" : "-Xmx1638m",
  993. "yarn.app.mapreduce.am.resource.mb" : "2048"
  994. }
  995. }
  996. },
  997. {
  998. "parquet-logging" : {
  999. "properties_attributes" : { },
  1000. "properties" : {
  1001. "content" : "\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n# Properties file which configures the operation of the JDK\n# logging facility.\n\n# The system will look for this config file, first using\n# a System property specified at startup:\n#\n# >java -Djava.util.logging.config.file=myLoggingConfigFilePath\n#\n# If this property is not specified, then the config file is\n# retrieved from its default location at:\n#\n# JDK_HOME/jre/lib/logging.properties\n\n# Global logging properties.\n# ------------------------------------------\n# The set of handlers to be loaded upon startup.\n# Comma-separated list of class names.\n# (? LogManager docs say no comma here, but JDK example has comma.)\n# handlers=java.util.logging.ConsoleHandler\norg.apache.parquet.handlers= java.util.logging.FileHandler\n\n# Default global logging level.\n# Loggers and Handlers may override this level\n.level=INFO\n\n# Handlers\n# -----------------------------------------\n\n# --- ConsoleHandler ---\n# Override of global logging level\njava.util.logging.ConsoleHandler.level=INFO\njava.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter\njava.util.logging.SimpleFormatter.format=[%1$tc] %4$s: %2$s - %5$s %6$s%n\n\n# --- FileHandler ---\n# Override of global logging level\njava.util.logging.FileHandler.level=ALL\n\n# Naming style for the output file:\n# (The output file is placed in the system temporary directory.\n# %u is used to provide unique identifier for the file.\n# For more information refer\n# https://docs.oracle.com/javase/7/docs/api/java/util/logging/FileHandler.html)\njava.util.logging.FileHandler.pattern=%t/parquet-%u.log\n\n# Limiting size of output file in bytes:\njava.util.logging.FileHandler.limit=50000000\n\n# Number of output files to cycle through, by appending an\n# integer to the base file name:\njava.util.logging.FileHandler.count=1\n\n# Style of output (Simple or XML):\njava.util.logging.FileHandler.formatter=java.util.logging.SimpleFormatter"
  1002. }
  1003. }
  1004. },
  1005. {
  1006. "hdfs-log4j" : {
  1007. "properties_attributes" : { },
  1008. "properties" : {
  1009. "hadoop_security_log_max_backup_size" : "256",
  1010. "hadoop_security_log_number_of_backup_files" : "20",
  1011. "hadoop_log_number_of_backup_files" : "10",
  1012. "content" : "\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied. See the License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n\n\n# Define some default values that can be overridden by system properties\n# To change daemon root logger use hadoop_root_logger in hadoop-env\nhadoop.root.logger=INFO,console\nhadoop.log.dir=.\nhadoop.log.file=hadoop.log\n\n\n# Define the root logger to the system property \"hadoop.root.logger\".\nlog4j.rootLogger=${hadoop.root.logger}, EventCounter\n\n# Logging Threshold\nlog4j.threshhold=ALL\n\n#\n# Daily Rolling File Appender\n#\n\nlog4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}\n\n# Rollver at midnight\nlog4j.appender.DRFA.DatePattern=.yyyy-MM-dd\n\n# 30-day backup\n#log4j.appender.DRFA.MaxBackupIndex=30\nlog4j.appender.DRFA.layout=org.apache.log4j.PatternLayout\n\n# Pattern format: Date LogLevel LoggerName LogMessage\nlog4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\n# Debugging Pattern format\n#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\n\n\n#\n# console\n# Add \"console\" to rootlogger above if you want to use this\n#\n\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\nlog4j.appender.console.target=System.err\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\nlog4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n\n\n#\n# TaskLog Appender\n#\n\n#Default values\nhadoop.tasklog.taskid=null\nhadoop.tasklog.iscleanup=false\nhadoop.tasklog.noKeepSplits=4\nhadoop.tasklog.totalLogFileSize=100\nhadoop.tasklog.purgeLogSplits=true\nhadoop.tasklog.logsRetainHours=12\n\nlog4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender\nlog4j.appender.TLA.taskId=${hadoop.tasklog.taskid}\nlog4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup}\nlog4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}\n\nlog4j.appender.TLA.layout=org.apache.log4j.PatternLayout\nlog4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\n\n#\n#Security audit appender\n#\nhadoop.security.logger=INFO,console\nhadoop.security.log.maxfilesize={{hadoop_security_log_max_backup_size}}MB\nhadoop.security.log.maxbackupindex={{hadoop_security_log_number_of_backup_files}}\nlog4j.category.SecurityLogger=${hadoop.security.logger}\nhadoop.security.log.file=SecurityAuth.audit\nlog4j.additivity.SecurityLogger=false\nlog4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}\nlog4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout\nlog4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\nlog4j.appender.DRFAS.DatePattern=.yyyy-MM-dd\n\nlog4j.appender.RFAS=org.apache.log4j.RollingFileAppender\nlog4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}\nlog4j.appender.RFAS.layout=org.apache.log4j.PatternLayout\nlog4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\nlog4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize}\nlog4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex}\n\n#\n# hdfs audit logging\n#\nhdfs.audit.logger=INFO,console\nlog4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger}\nlog4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false\nlog4j.appender.DRFAAUDIT=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.DRFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log\nlog4j.appender.DRFAAUDIT.layout=org.apache.log4j.PatternLayout\nlog4j.appender.DRFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\nlog4j.appender.DRFAAUDIT.DatePattern=.yyyy-MM-dd\n\n#\n# NameNode metrics logging.\n# The default is to retain two namenode-metrics.log files up to 64MB each.\n#\nnamenode.metrics.logger=INFO,NullAppender\nlog4j.logger.NameNodeMetricsLog=${namenode.metrics.logger}\nlog4j.additivity.NameNodeMetricsLog=false\nlog4j.appender.NNMETRICSRFA=org.apache.log4j.RollingFileAppender\nlog4j.appender.NNMETRICSRFA.File=${hadoop.log.dir}/namenode-metrics.log\nlog4j.appender.NNMETRICSRFA.layout=org.apache.log4j.PatternLayout\nlog4j.appender.NNMETRICSRFA.layout.ConversionPattern=%d{ISO8601} %m%n\nlog4j.appender.NNMETRICSRFA.MaxBackupIndex=1\nlog4j.appender.NNMETRICSRFA.MaxFileSize=64MB\n\n#\n# mapred audit logging\n#\nmapred.audit.logger=INFO,console\nlog4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger}\nlog4j.additivity.org.apache.hadoop.mapred.AuditLogger=false\nlog4j.appender.MRAUDIT=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log\nlog4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout\nlog4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\nlog4j.appender.MRAUDIT.DatePattern=.yyyy-MM-dd\n\n#\n# Rolling File Appender\n#\n\nlog4j.appender.RFA=org.apache.log4j.RollingFileAppender\nlog4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file}\n\n# Logfile size and and 30-day backups\nlog4j.appender.RFA.MaxFileSize={{hadoop_log_max_backup_size}}MB\nlog4j.appender.RFA.MaxBackupIndex={{hadoop_log_number_of_backup_files}}\n\nlog4j.appender.RFA.layout=org.apache.log4j.PatternLayout\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\n\n\n# Custom Logging levels\n\nhadoop.metrics.log.level=INFO\n#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG\n#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG\n#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG\nlog4j.logger.org.apache.hadoop.metrics2=${hadoop.metrics.log.level}\n\n# Jets3t library\nlog4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR\n\n#\n# Null Appender\n# Trap security logger on the hadoop client side\n#\nlog4j.appender.NullAppender=org.apache.log4j.varia.NullAppender\n\n#\n# Event Counter Appender\n# Sends counts of logging messages at different severity levels to Hadoop Metrics.\n#\nlog4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter\n\n# Removes \"deprecated\" messages\nlog4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN\n\n#\n# HDFS block state change log from block manager\n#\n# Uncomment the following to suppress normal block state change\n# messages from BlockManager in NameNode.\n#log4j.logger.BlockStateChange=WARN",
  1013. "hadoop_log_max_backup_size" : "256"
  1014. }
  1015. }
  1016. },
  1017. {
  1018. "hive-site" : {
  1019. "properties_attributes" : {
  1020. "hidden" : {
  1021. "javax.jdo.option.ConnectionPassword" : "HIVE_CLIENT,WEBHCAT_SERVER,HCAT,CONFIG_DOWNLOAD"
  1022. }
  1023. },
  1024. "properties" : {
  1025. "hive.exec.reducers.bytes.per.reducer" : "67108864",
  1026. "hive.metastore.pre.event.listeners" : "org.apache.hadoop.hive.ql.security.authorization.AuthorizationPreEventListener",
  1027. "hive.optimize.reducededuplication" : "true",
  1028. "hive.metastore.sasl.enabled" : "false",
  1029. "hive.vectorized.execution.enabled" : "true",
  1030. "hive.security.authorization.manager" : "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory",
  1031. "hive.auto.convert.join.noconditionaltask" : "true",
  1032. "hive.exec.orc.compression.strategy" : "SPEED",
  1033. "hive.server2.allow.user.substitution" : "true",
  1034. "hive.limit.pushdown.memory.usage" : "0.04",
  1035. "hive.merge.rcfile.block.level" : "true",
  1036. "hive.stats.fetch.column.stats" : "true",
  1037. "hive.tez.dynamic.partition.pruning" : "true",
  1038. "hive.exec.max.dynamic.partitions.pernode" : "2000",
  1039. "hive.exec.orc.default.compress" : "ZLIB",
  1040. "hive.merge.mapredfiles" : "false",
  1041. "hive.compactor.abortedtxn.threshold" : "1000",
  1042. "hive.map.aggr" : "true",
  1043. "hive.tez.dynamic.partition.pruning.max.data.size" : "104857600",
  1044. "hive.compactor.initiator.on" : "false",
  1045. "hive.smbjoin.cache.rows" : "10000",
  1046. "hive.exec.max.dynamic.partitions" : "5000",
  1047. "hive.auto.convert.join" : "true",
  1048. "hive.server2.support.dynamic.service.discovery" : "true",
  1049. "hive.tez.log.level" : "INFO",
  1050. "hive.compactor.worker.timeout" : "86400L",
  1051. "hive.custom-extensions.root" : "/hdp/ext/{{major_stack_version}}/hive",
  1052. "hive.metastore.authorization.storage.checks" : "false",
  1053. "hive.merge.mapfiles" : "true",
  1054. "hive.exec.post.hooks" : "org.apache.hadoop.hive.ql.hooks.ATSHook",
  1055. "hive.server2.transport.mode" : "binary",
  1056. "hive.server2.thrift.http.path" : "cliservice",
  1057. "hive.enforce.sortmergebucketmapjoin" : "true",
  1058. "hive.metastore.execute.setugi" : "true",
  1059. "hive.tez.smb.number.waves" : "0.5",
  1060. "hive.server2.authentication.spnego.keytab" : "HTTP/_HOST@EXAMPLE.COM",
  1061. "hive.zookeeper.quorum" : "%HOSTGROUP::host_group_1%:2181",
  1062. "hive.mapjoin.bucket.cache.size" : "10000",
  1063. "hive.metastore.kerberos.principal" : "hive/_HOST@EXAMPLE.COM",
  1064. "hive.tez.auto.reducer.parallelism" : "true",
  1065. "hive.security.metastore.authorization.manager" : "org.apache.hadoop.hive.ql.security.authorization.StorageBasedAuthorizationProvider",
  1066. "datanucleus.autoCreateSchema" : "false",
  1067. "datanucleus.fixedDatastore" : "true",
  1068. "hive.compute.query.using.stats" : "true",
  1069. "hive.security.authorization.enabled" : "false",
  1070. "hive.server2.thrift.sasl.qop" : "auth",
  1071. "hive.merge.orcfile.stripe.level" : "true",
  1072. "hive.orc.splits.include.file.footer" : "false",
  1073. "hive.exec.compress.output" : "false",
  1074. "hive.user.install.directory" : "/user/",
  1075. "hive.prewarm.enabled" : "false",
  1076. "hive.compactor.delta.num.threshold" : "10",
  1077. "hive.orc.compute.splits.num.threads" : "10",
  1078. "hive.vectorized.groupby.checkinterval" : "4096",
  1079. "hive.txn.manager" : "org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager",
  1080. "datanucleus.cache.level2.type" : "none",
  1081. "hive.map.aggr.hash.percentmemory" : "0.5",
  1082. "hive.optimize.bucketmapjoin" : "true",
  1083. "hive.tez.max.partition.factor" : "2.0",
  1084. "hive.server2.thrift.port" : "10000",
  1085. "hive.exec.failure.hooks" : "org.apache.hadoop.hive.ql.hooks.ATSHook",
  1086. "hive.exec.compress.intermediate" : "false",
  1087. "hive.exec.max.created.files" : "100000",
  1088. "hive.mapred.reduce.tasks.speculative.execution" : "false",
  1089. "hive.vectorized.groupby.flush.percent" : "0.1",
  1090. "hive.metastore.client.socket.timeout" : "1800s",
  1091. "hive.server2.tez.initialize.default.sessions" : "false",
  1092. "atlas.hook.hive.minThreads" : "1",
  1093. "hive.stats.autogather" : "true",
  1094. "hive.optimize.sort.dynamic.partition" : "false",
  1095. "hive.txn.max.open.batch" : "1000",
  1096. "hive.default.fileformat" : "TextFile",
  1097. "hive.mapjoin.optimized.hashtable" : "true",
  1098. "hive.vectorized.groupby.maxentries" : "100000",
  1099. "hive.compactor.check.interval" : "300L",
  1100. "hive.security.authenticator.manager" : "org.apache.hadoop.hive.ql.security.ProxyUserAuthenticator",
  1101. "hive.security.metastore.authorization.auth.reads" : "true",
  1102. "hive.compactor.delta.pct.threshold" : "0.1f",
  1103. "hive.vectorized.execution.reduce.enabled" : "false",
  1104. "javax.jdo.option.ConnectionDriverName" : "com.mysql.jdbc.Driver",
  1105. "hive.stats.dbclass" : "fs",
  1106. "hive.exec.parallel" : "false",
  1107. "hive.compactor.worker.threads" : "0",
  1108. "hive.exec.submitviachild" : "false",
  1109. "hive.fetch.task.conversion" : "more",
  1110. "hive.server2.authentication" : "NONE",
  1111. "hive.map.aggr.hash.force.flush.memory.threshold" : "0.9",
  1112. "hive.start.cleanup.scratchdir" : "false",
  1113. "hive.merge.tezfiles" : "false",
  1114. "hive.metastore.cache.pinobjtypes" : "Table,Database,Type,FieldSchema,Order",
  1115. "hive.optimize.index.filter" : "true",
  1116. "hive.server2.authentication.spnego.principal" : "/etc/security/keytabs/spnego.service.keytab",
  1117. "hive.convert.join.bucket.mapjoin.tez" : "false",
  1118. "hive.metastore.warehouse.dir" : "/apps/hive/warehouse",
  1119. "hive.execution.engine" : "tez",
  1120. "atlas.hook.hive.maxThreads" : "1",
  1121. "hive.tez.dynamic.partition.pruning.max.event.size" : "1048576",
  1122. "hive.cbo.enable" : "true",
  1123. "hive.exec.orc.encoding.strategy" : "SPEED",
  1124. "hive.optimize.constant.propagation" : "true",
  1125. "hive.tez.container.size" : "2048",
  1126. "hive.metastore.connect.retries" : "24",
  1127. "hive.optimize.reducededuplication.min.reducer" : "4",
  1128. "hive.tez.input.format" : "org.apache.hadoop.hive.ql.io.HiveInputFormat",
  1129. "hive.cluster.delegation.token.store.zookeeper.connectString" : "%HOSTGROUP::host_group_1%:2181",
  1130. "hive.metastore.uris" : "thrift://%HOSTGROUP::host_group_1%:9083",
  1131. "hive.server2.max.start.attempts" : "5",
  1132. "hive.exec.dynamic.partition.mode" : "strict",
  1133. "hive.server2.thrift.max.worker.threads" : "500",
  1134. "hive.server2.use.SSL" : "false",
  1135. "hive.tez.java.opts" : "-server -Djava.net.preferIPv4Stack=true -XX:NewRatio=8 -XX:+UseNUMA -XX:+UseParallelGC -XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps",
  1136. "hive.exec.submit.local.task.via.child" : "true",
  1137. "hive.optimize.null.scan" : "true",
  1138. "hive.exec.orc.default.stripe.size" : "67108864",
  1139. "hive.limit.optimize.enable" : "true",
  1140. "hive.cluster.delegation.token.store.zookeeper.znode" : "/hive/cluster/delegation",
  1141. "hive.exec.pre.hooks" : "org.apache.hadoop.hive.ql.hooks.ATSHook",
  1142. "hive.cluster.delegation.token.store.class" : "org.apache.hadoop.hive.thrift.ZooKeeperTokenStore",
  1143. "ambari.hive.db.schema.name" : "hive",
  1144. "hive.zookeeper.client.port" : "2181",
  1145. "hive.enforce.sorting" : "true",
  1146. "hive.tez.cpu.vcores" : "-1",
  1147. "hive.metastore.client.connect.retry.delay" : "5s",
  1148. "hive.server2.tez.default.queues" : "default",
  1149. "hive.server2.tez.sessions.per.default.queue" : "1",
  1150. "hive.server2.thrift.http.port" : "10001",
  1151. "hive.server2.logging.operation.log.location" : "/tmp/hive/operation_logs",
  1152. "javax.jdo.option.ConnectionURL" : "jdbc:mysql://%HOSTGROUP::host_group_1%/hive?createDatabaseIfNotExist=true",
  1153. "hive.map.aggr.hash.min.reduction" : "0.5",
  1154. "hive.merge.size.per.task" : "256000000",
  1155. "hive.merge.smallfiles.avgsize" : "16000000",
  1156. "hive.exec.reducers.max" : "1009",
  1157. "hive.optimize.metadataonly" : "true",
  1158. "hive.fetch.task.conversion.threshold" : "1073741824",
  1159. "hive.prewarm.numcontainers" : "3",
  1160. "hive.tez.min.partition.factor" : "0.25",
  1161. "hive.auto.convert.join.noconditionaltask.size" : "572662306",
  1162. "hive.server2.logging.operation.enabled" : "true",
  1163. "hive.metastore.kerberos.keytab.file" : "/etc/security/keytabs/hive.service.keytab",
  1164. "hive.exec.parallel.thread.number" : "8",
  1165. "hive.support.concurrency" : "false",
  1166. "javax.jdo.option.ConnectionUserName" : "hive",
  1167. "hive.conf.restricted.list" : "hive.security.authenticator.manager,hive.security.authorization.manager,hive.users.in.admin.role",
  1168. "hive.auto.convert.sortmerge.join.to.mapjoin" : "false",
  1169. "hive.fetch.task.aggr" : "false",
  1170. "hive.cli.print.header" : "false",
  1171. "hive.server2.table.type.mapping" : "CLASSIC",
  1172. "hive.txn.timeout" : "300",
  1173. "hive.warehouse.subdir.inherit.perms" : "true",
  1174. "hive.stats.fetch.partition.stats" : "true",
  1175. "hive.optimize.bucketmapjoin.sortedmerge" : "false",
  1176. "hive.security.metastore.authenticator.manager" : "org.apache.hadoop.hive.ql.security.HadoopDefaultMetastoreAuthenticator",
  1177. "hive.server2.enable.doAs" : "true",
  1178. "hive.server2.zookeeper.namespace" : "hiveserver2",
  1179. "hive.default.fileformat.managed" : "TextFile",
  1180. "hive.enforce.bucketing" : "true",
  1181. "hive.exec.scratchdir" : "/tmp/hive",
  1182. "hive.exec.dynamic.partition" : "true",
  1183. "hive.metastore.server.max.threads" : "100000",
  1184. "hive.metastore.failure.retries" : "24",
  1185. "hive.auto.convert.sortmerge.join" : "true",
  1186. "hive.zookeeper.namespace" : "hive_zookeeper_namespace"
  1187. }
  1188. }
  1189. },
  1190. {
  1191. "activity-conf" : {
  1192. "properties_attributes" : { },
  1193. "properties" : {
  1194. "global.activity.processing.parallelism" : "8",
  1195. "yarn_app.activity.watcher.enabled" : "true",
  1196. "mr_job.max.job.size.mb.for.parallel.execution" : "500",
  1197. "global.activity.processor.pool.max.wait.seconds" : "60",
  1198. "phoenix.sink.batch.size" : "100",
  1199. "phoenix.sink.flush.interval.seconds" : "30",
  1200. "hdfs.activity.watcher.enabled" : "true",
  1201. "global.activity.analyzer.user" : "activity_analyzer",
  1202. "tez_job.activity.watcher.enabled" : "true",
  1203. "activity.explorer.user" : "activity_explorer",
  1204. "mr_job.activity.watcher.enabled" : "true",
  1205. "tez_job.tmp.dir" : "/var/lib/smartsense/activity-analyzer/tez/tmp/"
  1206. }
  1207. }
  1208. },
  1209. {
  1210. "tez-site" : {
  1211. "properties_attributes" : { },
  1212. "properties" : {
  1213. "tez.am.launch.cmd-opts" : "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC{{heap_dump_opts}}",
  1214. "tez.runtime.sorter.class" : "PIPELINED",
  1215. "tez.counters.max.groups" : "3000",
  1216. "tez.am.view-acls" : "*",
  1217. "tez.runtime.shuffle.memory.limit.percent" : "0.25",
  1218. "tez.staging-dir" : "/tmp/${user.name}/staging",
  1219. "tez.am.container.reuse.locality.delay-allocation-millis" : "250",
  1220. "tez.runtime.compress" : "true",
  1221. "tez.am.am-rm.heartbeat.interval-ms.max" : "250",
  1222. "tez.am.container.idle.release-timeout-min.millis" : "10000",
  1223. "tez.task.launch.cmd-opts" : "-XX:+PrintGCDetails -verbose:gc -XX:+PrintGCTimeStamps -XX:+UseNUMA -XX:+UseParallelGC{{heap_dump_opts}}",
  1224. "tez.lib.uris" : "/hdp/apps/${hdp.version}/tez/tez.tar.gz",
  1225. "tez.counters.max" : "10000",
  1226. "tez.generate.debug.artifacts" : "false",
  1227. "tez.runtime.convert.user-payload.to.history-text" : "false",
  1228. "tez.shuffle-vertex-manager.max-src-fraction" : "0.4",
  1229. "tez.am.log.level" : "INFO",
  1230. "tez.task.am.heartbeat.counter.interval-ms.max" : "4000",
  1231. "tez.am.container.idle.release-timeout-max.millis" : "20000",
  1232. "tez.am.resource.memory.mb" : "4096",
  1233. "tez.am.max.app.attempts" : "2",
  1234. "tez.cluster.additional.classpath.prefix" : "/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure",
  1235. "tez.am.container.reuse.non-local-fallback.enabled" : "false",
  1236. "tez.am.container.reuse.rack-fallback.enabled" : "true",
  1237. "tez.task.get-task.sleep.interval-ms.max" : "200",
  1238. "tez.runtime.io.sort.mb" : "540",
  1239. "tez.task.resource.memory.mb" : "2048",
  1240. "tez.runtime.optimize.local.fetch" : "true",
  1241. "tez.runtime.pipelined.sorter.sort.threads" : "2",
  1242. "tez.shuffle-vertex-manager.min-src-fraction" : "0.2",
  1243. "tez.session.am.dag.submit.timeout.secs" : "600",
  1244. "tez.session.client.timeout.secs" : "-1",
  1245. "tez.am.launch.cluster-default.cmd-opts" : "-server -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}",
  1246. "tez.am.maxtaskfailures.per.node" : "10",
  1247. "tez.am.tez-ui.history-url.template" : "__HISTORY_URL_BASE__?viewPath=%2F%23%2Ftez-app%2F__APPLICATION_ID__",
  1248. "tez.history.logging.timeline-cache-plugin.old-num-dags-per-group" : "5",
  1249. "tez.runtime.shuffle.fetch.buffer.percent" : "0.6",
  1250. "tez.task.launch.env" : "LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64",
  1251. "tez.use.cluster.hadoop-libs" : "false",
  1252. "tez.queue.name" : "default",
  1253. "tez.runtime.unordered.output.buffer.size-mb" : "153",
  1254. "tez.am.container.reuse.enabled" : "true",
  1255. "tez.grouping.split-waves" : "1.7",
  1256. "tez.grouping.max-size" : "1073741824",
  1257. "tez.task.launch.cluster-default.cmd-opts" : "-server -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}",
  1258. "tez.task.max-events-per-heartbeat" : "500",
  1259. "tez.history.logging.service.class" : "org.apache.tez.dag.history.logging.ats.ATSV15HistoryLoggingService",
  1260. "tez.grouping.min-size" : "16777216",
  1261. "tez.task.generate.counters.per.io" : "true",
  1262. "tez.am.launch.env" : "LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64",
  1263. "tez.runtime.compress.codec" : "org.apache.hadoop.io.compress.SnappyCodec"
  1264. }
  1265. }
  1266. },
  1267. {
  1268. "activity-zeppelin-site" : {
  1269. "properties_attributes" : { },
  1270. "properties" : {
  1271. "zeppelin.notebook.dir" : "/var/lib/smartsense/activity-explorer/notebook",
  1272. "zeppelin.server.context.path" : "/",
  1273. "zeppelin.ssl.keystore.type" : "JKS",
  1274. "zeppelin.ssl.truststore.path" : "/var/lib/smartsense/activity-explorer/truststore",
  1275. "zeppelin.war.tempdir" : "/var/lib/smartsense/activity-explorer/webapp",
  1276. "zeppelin.websocket.max.text.message.size" : "1024000",
  1277. "zeppelin.notebook.homescreen.hide" : "false",
  1278. "zeppelin.interpreters" : "org.apache.zeppelin.phoenix.PhoenixInterpreter",
  1279. "zeppelin.server.port" : "9060",
  1280. "zeppelin.ssl" : "false",
  1281. "zeppelin.notebook.storage" : "org.apache.zeppelin.notebook.repo.VFSNotebookRepo",
  1282. "zeppelin.interpreter.connect.timeout" : "30000",
  1283. "zeppelin.anonymous.allowed" : "false",
  1284. "zeppelin.interpreter.dir" : "/usr/hdp/share/hst/activity-explorer/interpreter",
  1285. "zeppelin.ssl.truststore.type" : "JKS",
  1286. "zeppelin.ssl.keystore.path" : "/var/lib/smartsense/activity-explorer/keystore",
  1287. "zeppelin.server.allowed.origins" : "*",
  1288. "zeppelin.server.addr" : "0.0.0.0",
  1289. "zeppelin.ssl.client.auth" : "false"
  1290. }
  1291. }
  1292. },
  1293. {
  1294. "webhcat-site" : {
  1295. "properties_attributes" : { },
  1296. "properties" : {
  1297. "templeton.zookeeper.hosts" : "%HOSTGROUP::host_group_1%:2181",
  1298. "webhcat.proxyuser.root.groups" : "*",
  1299. "webhcat.proxyuser.root.hosts" : "sandbox-hdp.hortonworks.com",
  1300. "templeton.hadoop.queue.name" : "default",
  1301. "templeton.hadoop" : "/usr/hdp/${hdp.version}/hadoop/bin/hadoop",
  1302. "templeton.hive.extra.files" : "/usr/hdp/${hdp.version}/tez/conf/tez-site.xml,/usr/hdp/${hdp.version}/tez,/usr/hdp/${hdp.version}/tez/lib",
  1303. "templeton.hcat.home" : "hive.tar.gz/hive/hcatalog",
  1304. "templeton.override.enabled" : "false",
  1305. "templeton.hive.properties" : "hive.metastore.local=false,hive.metastore.uris=thrift://%HOSTGROUP::host_group_1%:9083,hive.metastore.sasl.enabled=false,hive.metastore.execute.setugi=true",
  1306. "templeton.python" : "${env.PYTHON_CMD}",
  1307. "templeton.storage.class" : "org.apache.hive.hcatalog.templeton.tool.ZooKeeperStorage",
  1308. "templeton.streaming.jar" : "hdfs:///hdp/apps/${hdp.version}/mapreduce/hadoop-streaming.jar",
  1309. "templeton.hadoop.conf.dir" : "/etc/hadoop/conf",
  1310. "templeton.hcat" : "/usr/hdp/${hdp.version}/hive/bin/hcat",
  1311. "templeton.jar" : "/usr/hdp/${hdp.version}/hive/share/webhcat/svr/lib/hive-webhcat-*.jar",
  1312. "templeton.port" : "50111",
  1313. "templeton.libjars" : "/usr/hdp/${hdp.version}/zookeeper/zookeeper.jar,/usr/hdp/${hdp.version}/hive/lib/hive-common.jar",
  1314. "templeton.hive.path" : "hive.tar.gz/hive/bin/hive",
  1315. "templeton.exec.timeout" : "60000",
  1316. "templeton.sqoop.home" : "sqoop.tar.gz/sqoop",
  1317. "templeton.sqoop.archive" : "hdfs:///hdp/apps/${hdp.version}/sqoop/sqoop.tar.gz",
  1318. "templeton.hive.archive" : "hdfs:///hdp/apps/${hdp.version}/hive/hive.tar.gz",
  1319. "templeton.hive.home" : "hive.tar.gz/hive",
  1320. "templeton.pig.archive" : "hdfs:///hdp/apps/${hdp.version}/pig/pig.tar.gz",
  1321. "templeton.pig.path" : "pig.tar.gz/pig/bin/pig",
  1322. "templeton.sqoop.path" : "sqoop.tar.gz/sqoop/bin/sqoop"
  1323. }
  1324. }
  1325. },
  1326. {
  1327. "slider-env" : {
  1328. "properties_attributes" : { },
  1329. "properties" : {
  1330. "content" : "\n# Set Slider-specific environment variables here.\n\n# The only required environment variable is JAVA_HOME. All others are\n# optional. When running a distributed configuration it is best to\n# set JAVA_HOME in this file, so that it is correctly defined on\n# remote nodes.\n\n# The java implementation to use. Required.\nexport JAVA_HOME={{java64_home}}\n# The hadoop conf directory. Optional as slider-client.xml can be edited to add properties.\nexport HADOOP_CONF_DIR={{hadoop_conf_dir}}"
  1331. }
  1332. }
  1333. },
  1334. {
  1335. "ams-hbase-env" : {
  1336. "properties_attributes" : { },
  1337. "properties" : {
  1338. "hbase_master_maxperm_size" : "128",
  1339. "max_open_files_limit" : "32768",
  1340. "regionserver_xmn_size" : "128",
  1341. "hbase_master_heapsize" : "640",
  1342. "hbase_regionserver_xmn_ratio" : "0.2",
  1343. "hbase_master_xmn_size" : "192",
  1344. "content" : "\n# Set environment variables here.\n\n# The java implementation to use. Java 1.6+ required.\nexport JAVA_HOME={{java64_home}}\n\n# HBase Configuration directory\nexport HBASE_CONF_DIR=${HBASE_CONF_DIR:-{{hbase_conf_dir}}}\n\n# Extra Java CLASSPATH elements. Optional.\nadditional_cp={{hbase_classpath_additional}}\nif [ -n \"$additional_cp\" ];\nthen\n export HBASE_CLASSPATH=${HBASE_CLASSPATH}:$additional_cp\nelse\n export HBASE_CLASSPATH=${HBASE_CLASSPATH}\nfi\n\n# The maximum amount of heap to use for hbase shell.\nexport HBASE_SHELL_OPTS=\"-Xmx256m\"\n\n# Extra Java runtime options.\n# Below are what we set by default. May only work with SUN JVM.\n# For more on why as well as other possible settings,\n# see http://wiki.apache.org/hadoop/PerformanceTuning\nexport HBASE_OPTS=\"-XX:+UseConcMarkSweepGC -XX:ErrorFile={{hbase_log_dir}}/hs_err_pid%p.log -Djava.io.tmpdir={{hbase_tmp_dir}}\"\nexport SERVER_GC_OPTS=\"-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:{{hbase_log_dir}}/gc.log-`date +'%Y%m%d%H%M'`\"\n# Uncomment below to enable java garbage collection logging.\n# export HBASE_OPTS=\"$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log\"\n\n# Uncomment and adjust to enable JMX exporting\n# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access.\n# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html\n#\n# export HBASE_JMX_BASE=\"-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false\"\n\n{% if java_version < 8 %}\nexport HBASE_MASTER_OPTS=\" -XX:PermSize=64m -XX:MaxPermSize={{hbase_master_maxperm_size}} -Xms{{hbase_heapsize}} -Xmx{{hbase_heapsize}} -Xmn{{hbase_master_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly\"\nexport HBASE_REGIONSERVER_OPTS=\"-XX:MaxPermSize=128m -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}\"\n{% else %}\nexport HBASE_MASTER_OPTS=\" -Xms{{hbase_heapsize}} -Xmx{{hbase_heapsize}} -Xmn{{hbase_master_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly\"\nexport HBASE_REGIONSERVER_OPTS=\" -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}\"\n{% endif %}\n\n\n# export HBASE_THRIFT_OPTS=\"$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103\"\n# export HBASE_ZOOKEEPER_OPTS=\"$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104\"\n\n# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default.\nexport HBASE_REGIONSERVERS=${HBASE_CONF_DIR}/regionservers\n\n# Extra ssh options. Empty by default.\n# export HBASE_SSH_OPTS=\"-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR\"\n\n# Where log files are stored. $HBASE_HOME/logs by default.\nexport HBASE_LOG_DIR={{hbase_log_dir}}\n\n# A string representing this instance of hbase. $USER by default.\n# export HBASE_IDENT_STRING=$USER\n\n# The scheduling priority for daemon processes. See 'man nice'.\n# export HBASE_NICENESS=10\n\n# The directory where pid files are stored. /tmp by default.\nexport HBASE_PID_DIR={{hbase_pid_dir}}\n\n# Seconds to sleep between slave commands. Unset by default. This\n# can be useful in large clusters, where, e.g., slave rsyncs can\n# otherwise arrive faster than the master can service them.\n# export HBASE_SLAVE_SLEEP=0.1\n\n# Tell HBase whether it should manage it's own instance of Zookeeper or not.\nexport HBASE_MANAGES_ZK=false\n\n{% if security_enabled %}\nexport HBASE_OPTS=\"$HBASE_OPTS -Djava.security.auth.login.config={{client_jaas_config_file}}\"\nexport HBASE_MASTER_OPTS=\"$HBASE_MASTER_OPTS -Djava.security.auth.login.config={{master_jaas_config_file}} -Djavax.security.auth.useSubjectCredsOnly=false\"\nexport HBASE_REGIONSERVER_OPTS=\"$HBASE_REGIONSERVER_OPTS -Djava.security.auth.login.config={{regionserver_jaas_config_file}} -Djavax.security.auth.useSubjectCredsOnly=false\"\nexport HBASE_ZOOKEEPER_OPTS=\"$HBASE_ZOOKEEPER_OPTS -Djava.security.auth.login.config={{ams_zookeeper_jaas_config_file}}\"\n{% endif %}\n\n# use embedded native libs\n_HADOOP_NATIVE_LIB=\"/usr/lib/ams-hbase/lib/hadoop-native/\"\nexport HBASE_OPTS=\"$HBASE_OPTS -Djava.library.path=${_HADOOP_NATIVE_LIB}\"\n\n# Unset HADOOP_HOME to avoid importing HADOOP installed cluster related configs like: /usr/hdp/2.2.0.0-2041/hadoop/conf/\nexport HADOOP_HOME={{ams_hbase_home_dir}}\n\n# Explicitly Setting HBASE_HOME for AMS HBase so that there is no conflict\nexport HBASE_HOME={{ams_hbase_home_dir}}",
  1345. "hbase_pid_dir" : "/var/run/ambari-metrics-collector/",
  1346. "hbase_regionserver_shutdown_timeout" : "30",
  1347. "hbase_log_dir" : "/var/log/ambari-metrics-collector",
  1348. "hbase_regionserver_heapsize" : "768",
  1349. "hbase_classpath_additional" : ""
  1350. }
  1351. }
  1352. },
  1353. {
  1354. "ams-log4j" : {
  1355. "properties_attributes" : { },
  1356. "properties" : {
  1357. "ams_log_number_of_backup_files" : "60",
  1358. "ams_log_max_backup_size" : "80",
  1359. "content" : "\n#\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n#\n\n# Define some default values that can be overridden by system properties\nams.log.dir=.\nams.log.file=ambari-metrics-collector.log\n\n# Root logger option\nlog4j.rootLogger=INFO,file\n\n# Direct log messages to a log file\nlog4j.appender.file=org.apache.log4j.RollingFileAppender\nlog4j.appender.file.File=${ams.log.dir}/${ams.log.file}\nlog4j.appender.file.MaxFileSize={{ams_log_max_backup_size}}MB\nlog4j.appender.file.MaxBackupIndex={{ams_log_number_of_backup_files}}\nlog4j.appender.file.layout=org.apache.log4j.PatternLayout\nlog4j.appender.file.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n"
  1360. }
  1361. }
  1362. },
  1363. {
  1364. "hivemetastore-site" : {
  1365. "properties_attributes" : { },
  1366. "properties" : {
  1367. "hive.service.metrics.hadoop2.component" : "hivemetastore",
  1368. "hive.metastore.metrics.enabled" : "true",
  1369. "hive.service.metrics.reporter" : "HADOOP2"
  1370. }
  1371. }
  1372. },
  1373. {
  1374. "ams-ssl-server" : {
  1375. "properties_attributes" : { },
  1376. "properties" : {
  1377. "ssl.server.truststore.location" : "/etc/security/serverKeys/all.jks",
  1378. "ssl.server.truststore.reload.interval" : "10000",
  1379. "ssl.server.truststore.type" : "jks",
  1380. "ssl.server.keystore.location" : "/etc/security/serverKeys/keystore.jks",
  1381. "ssl.server.keystore.type" : "jks"
  1382. }
  1383. }
  1384. },
  1385. {
  1386. "hive-exec-log4j2" : {
  1387. "properties_attributes" : { },
  1388. "properties" : {
  1389. "content" : "\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nstatus = INFO\nname = HiveExecLog4j2\npackages = org.apache.hadoop.hive.ql.log\n\n# list of properties\nproperty.hive.log.level = {{hive_log_level}}\nproperty.hive.root.logger = FA\nproperty.hive.query.id = hadoop\nproperty.hive.log.dir = ${sys:java.io.tmpdir}/${sys:user.name}\nproperty.hive.log.file = ${sys:hive.query.id}.log\n\n# list of all appenders\nappenders = console, FA\n\n# console appender\nappender.console.type = Console\nappender.console.name = console\nappender.console.target = SYSTEM_ERR\nappender.console.layout.type = PatternLayout\nappender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n\n\n# simple file appender\nappender.FA.type = File\nappender.FA.name = FA\nappender.FA.fileName = ${sys:hive.log.dir}/${sys:hive.log.file}\nappender.FA.layout.type = PatternLayout\nappender.FA.layout.pattern = %d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n\n\n# list of all loggers\nloggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX\n\nlogger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn\nlogger.NIOServerCnxn.level = WARN\n\nlogger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO\nlogger.ClientCnxnSocketNIO.level = WARN\n\nlogger.DataNucleus.name = DataNucleus\nlogger.DataNucleus.level = ERROR\n\nlogger.Datastore.name = Datastore\nlogger.Datastore.level = ERROR\n\nlogger.JPOX.name = JPOX\nlogger.JPOX.level = ERROR\n\n# root logger\nrootLogger.level = ${sys:hive.log.level}\nrootLogger.appenderRefs = root\nrootLogger.appenderRef.root.ref = ${sys:hive.root.logger}"
  1390. }
  1391. }
  1392. },
  1393. {
  1394. "ams-hbase-log4j" : {
  1395. "properties_attributes" : { },
  1396. "properties" : {
  1397. "ams_hbase_log_maxfilesize" : "256",
  1398. "content" : "\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\n# Define some default values that can be overridden by system properties\nhbase.root.logger=INFO,console\nhbase.security.logger=INFO,console\nhbase.log.dir=.\nhbase.log.file=hbase.log\n\n# Define the root logger to the system property \"hbase.root.logger\".\nlog4j.rootLogger=${hbase.root.logger}\n\n# Logging Threshold\nlog4j.threshold=ALL\n\n#\n# Daily Rolling File Appender\n#\nlog4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file}\n\n# Rollver at midnight\nlog4j.appender.DRFA.DatePattern=.yyyy-MM-dd\n\n# 30-day backup\n#log4j.appender.DRFA.MaxBackupIndex=30\nlog4j.appender.DRFA.layout=org.apache.log4j.PatternLayout\n\n# Pattern format: Date LogLevel LoggerName LogMessage\nlog4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n\n\n# Rolling File Appender properties\nhbase.log.maxfilesize={{ams_hbase_log_maxfilesize}}MB\nhbase.log.maxbackupindex={{ams_hbase_log_maxbackupindex}}\n\n# Rolling File Appender\nlog4j.appender.RFA=org.apache.log4j.RollingFileAppender\nlog4j.appender.RFA.File=${hbase.log.dir}/${hbase.log.file}\n\nlog4j.appender.RFA.MaxFileSize=${hbase.log.maxfilesize}\nlog4j.appender.RFA.MaxBackupIndex=${hbase.log.maxbackupindex}\n\nlog4j.appender.RFA.layout=org.apache.log4j.PatternLayout\nlog4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n\n\n#\n# Security audit appender\n#\nhbase.security.log.file=SecurityAuth.audit\nhbase.security.log.maxfilesize={{ams_hbase_security_log_maxfilesize}}MB\nhbase.security.log.maxbackupindex={{ams_hbase_security_log_maxbackupindex}}\nlog4j.appender.RFAS=org.apache.log4j.RollingFileAppender\nlog4j.appender.RFAS.File=${hbase.log.dir}/${hbase.security.log.file}\nlog4j.appender.RFAS.MaxFileSize=${hbase.security.log.maxfilesize}\nlog4j.appender.RFAS.MaxBackupIndex=${hbase.security.log.maxbackupindex}\nlog4j.appender.RFAS.layout=org.apache.log4j.PatternLayout\nlog4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n\nlog4j.category.SecurityLogger=${hbase.security.logger}\nlog4j.additivity.SecurityLogger=false\n#log4j.logger.SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController=TRACE\n\n#\n# Null Appender\n#\nlog4j.appender.NullAppender=org.apache.log4j.varia.NullAppender\n\n#\n# console\n# Add \"console\" to rootlogger above if you want to use this\n#\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\nlog4j.appender.console.target=System.err\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\nlog4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %m%n\n\n# Custom Logging levels\n\nlog4j.logger.org.apache.zookeeper=INFO\n#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG\nlog4j.logger.org.apache.hadoop.hbase=INFO\n# Make these two classes INFO-level. Make them DEBUG to see more zk debug.\nlog4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO\nlog4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher=INFO\n#log4j.logger.org.apache.hadoop.dfs=DEBUG\n# Set this class to log INFO only otherwise its OTT\n# Enable this to get detailed connection error/retry logging.\n# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=TRACE\n\n\n# Uncomment this line to enable tracing on _every_ RPC call (this can be a lot of output)\n#log4j.logger.org.apache.hadoop.ipc.HBaseServer.trace=DEBUG\n\n# Uncomment the below if you want to remove logging of client region caching'\n# and scan of .META. messages\n# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=INFO\n# log4j.logger.org.apache.hadoop.hbase.client.MetaScanner=INFO",
  1399. "ams_hbase_log_maxbackupindex" : "20",
  1400. "ams_hbase_security_log_maxbackupindex" : "20",
  1401. "ams_hbase_security_log_maxfilesize" : "256"
  1402. }
  1403. }
  1404. },
  1405. {
  1406. "ssl-client" : {
  1407. "properties_attributes" : { },
  1408. "properties" : {
  1409. "ssl.client.truststore.reload.interval" : "10000",
  1410. "ssl.client.keystore.location" : "/etc/security/clientKeys/keystore.jks",
  1411. "ssl.client.truststore.location" : "/etc/security/clientKeys/all.jks",
  1412. "ssl.client.truststore.type" : "jks",
  1413. "ssl.client.keystore.type" : "jks"
  1414. }
  1415. }
  1416. },
  1417. {
  1418. "ams-ssl-client" : {
  1419. "properties_attributes" : { },
  1420. "properties" : {
  1421. "ssl.client.truststore.location" : "/etc/security/clientKeys/all.jks",
  1422. "ssl.client.truststore.type" : "jks",
  1423. "ssl.client.truststore.alias" : ""
  1424. }
  1425. }
  1426. },
  1427. {
  1428. "hive-atlas-application.properties" : {
  1429. "properties_attributes" : { },
  1430. "properties" : {
  1431. "atlas.hook.hive.maxThreads" : "5",
  1432. "atlas.hook.hive.keepAliveTime" : "10",
  1433. "atlas.hook.hive.synchronous" : "false",
  1434. "atlas.hook.hive.minThreads" : "5",
  1435. "atlas.hook.hive.numRetries" : "3",
  1436. "atlas.hook.hive.queueSize" : "1000"
  1437. }
  1438. }
  1439. },
  1440. {
  1441. "ams-hbase-policy" : {
  1442. "properties_attributes" : { },
  1443. "properties" : {
  1444. "security.masterregion.protocol.acl" : "*",
  1445. "security.admin.protocol.acl" : "*",
  1446. "security.client.protocol.acl" : "*"
  1447. }
  1448. }
  1449. },
  1450. {
  1451. "ranger-hdfs-security" : {
  1452. "properties_attributes" : { },
  1453. "properties" : { }
  1454. }
  1455. },
  1456. {
  1457. "ams-grafana-ini" : {
  1458. "properties_attributes" : { },
  1459. "properties" : {
  1460. "protocol" : "http",
  1461. "cert_key" : "/etc/ambari-metrics-grafana/conf/ams-grafana.key",
  1462. "port" : "3000",
  1463. "cert_file" : "/etc/ambari-metrics-grafana/conf/ams-grafana.crt",
  1464. "ca_cert" : "",
  1465. "content" : "\n##################### Grafana Configuration Example #####################\n#\n# Everything has defaults so you only need to uncomment things you want to\n# change\n\n# possible values : production, development\n; app_mode = production\n\n#################################### Paths ####################################\n[paths]\n# Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used)\n#\n;data = /var/lib/grafana\ndata = {{ams_grafana_data_dir}}\n#\n# Directory where grafana can store logs\n#\n;logs = /var/log/grafana\nlogs = {{ams_grafana_log_dir}}\n\n\n#################################### Server ####################################\n[server]\n# Protocol (http or https)\n;protocol = http\nprotocol = {{ams_grafana_protocol}}\n# The ip address to bind to, empty will bind to all interfaces\n;http_addr =\n\n# The http port to use\n;http_port = 3000\nhttp_port = {{ams_grafana_port}}\n\n# The public facing domain name used to access grafana from a browser\n;domain = localhost\n\n# Redirect to correct domain if host header does not match domain\n# Prevents DNS rebinding attacks\n;enforce_domain = false\n\n# The full public facing url\n;root_url = %(protocol)s://%(domain)s:%(http_port)s/\n\n# Log web requests\n;router_logging = false\n\n# the path relative working path\n;static_root_path = public\nstatic_root_path = /usr/lib/ambari-metrics-grafana/public\n\n# enable gzip\n;enable_gzip = false\n\n# https certs & key file\n;cert_file =\n;cert_key =\ncert_file = {{ams_grafana_cert_file}}\ncert_key = {{ams_grafana_cert_key}}\n\n#################################### Database ####################################\n[database]\n# Either \"mysql\", \"postgres\" or \"sqlite3\", it's your choice\n;type = sqlite3\n;host = 127.0.0.1:3306\n;name = grafana\n;user = root\n;password =\n\n# For \"postgres\" only, either \"disable\", \"require\" or \"verify-full\"\n;ssl_mode = disable\n\n# For \"sqlite3\" only, path relative to data_path setting\n;path = grafana.db\n\n#################################### Session ####################################\n[session]\n# Either \"memory\", \"file\", \"redis\", \"mysql\", \"postgres\", default is \"file\"\n;provider = file\n\n# Provider config options\n# memory: not have any config yet\n# file: session dir path, is relative to grafana data_path\n# redis: config like redis server e.g. `addr=127.0.0.1:6379,pool_size=100,db=grafana`\n# mysql: go-sql-driver/mysql dsn config string, e.g. `user:password@tcp(127.0.0.1:3306)/database_name`\n# postgres: user=a password=b host=localhost port=5432 dbname=c sslmode=disable\n;provider_config = sessions\n\n# Session cookie name\n;cookie_name = grafana_sess\n\n# If you use session in https only, default is false\n;cookie_secure = false\n\n# Session life time, default is 86400\n;session_life_time = 86400\n\n#################################### Analytics ####################################\n[analytics]\n# Server reporting, sends usage counters to stats.grafana.org every 24 hours.\n# No ip addresses are being tracked, only simple counters to track\n# running instances, dashboard and error counts. It is very helpful to us.\n# Change this option to false to disable reporting.\n;reporting_enabled = true\n\n# Google Analytics universal tracking code, only enabled if you specify an id here\n;google_analytics_ua_id =\n\n#################################### Security ####################################\n[security]\n# default admin user, created on startup\nadmin_user = {{ams_grafana_admin_user}}\n\n# default admin password, can be changed before first start of grafana, or in profile settings\n;admin_password =\n\n# used for signing\n;secret_key = SW2YcwTIb9zpOOhoPsMm\n\n# Auto-login remember days\n;login_remember_days = 7\n;cookie_username = grafana_user\n;cookie_remember_name = grafana_remember\n\n# disable gravatar profile images\n;disable_gravatar = false\n\n# data source proxy whitelist (ip_or_domain:port seperated by spaces)\n;data_source_proxy_whitelist =\n\n#################################### Users ####################################\n[users]\n# disable user signup / registration\n;allow_sign_up = true\n\n# Allow non admin users to create organizations\n;allow_org_create = true\n\n# Set to true to automatically assign new users to the default organization (id 1)\n;auto_assign_org = true\n\n# Default role new users will be automatically assigned (if disabled above is set to true)\n;auto_assign_org_role = Viewer\n\n# Background text for the user field on the login page\n;login_hint = email or username\n\n#################################### Anonymous Auth ##########################\n[auth.anonymous]\n# enable anonymous access\nenabled = true\n\n# specify organization name that should be used for unauthenticated users\norg_name = Main Org.\n\n# specify role for unauthenticated users\n;org_role = Admin\n\n#################################### Github Auth ##########################\n[auth.github]\n;enabled = false\n;allow_sign_up = false\n;client_id = some_id\n;client_secret = some_secret\n;scopes = user:email,read:org\n;auth_url = https://github.com/login/oauth/authorize\n;token_url = https://github.com/login/oauth/access_token\n;api_url = https://api.github.com/user\n;team_ids =\n;allowed_organizations =\n\n#################################### Google Auth ##########################\n[auth.google]\n;enabled = false\n;allow_sign_up = false\n;client_id = some_client_id\n;client_secret = some_client_secret\n;scopes = https://www.googleapis.com/auth/userinfo.profile https://www.googleapis.com/auth/userinfo.email\n;auth_url = https://accounts.google.com/o/oauth2/auth\n;token_url = https://accounts.google.com/o/oauth2/token\n;api_url = https://www.googleapis.com/oauth2/v1/userinfo\n;allowed_domains =\n\n#################################### Auth Proxy ##########################\n[auth.proxy]\n;enabled = false\n;header_name = X-WEBAUTH-USER\n;header_property = username\n;auto_sign_up = true\n\n#################################### Basic Auth ##########################\n[auth.basic]\n;enabled = true\n\n#################################### Auth LDAP ##########################\n[auth.ldap]\n;enabled = false\n;config_file = /etc/grafana/ldap.toml\n\n#################################### SMTP / Emailing ##########################\n[smtp]\n;enabled = false\n;host = localhost:25\n;user =\n;password =\n;cert_file =\n;key_file =\n;skip_verify = false\n;from_address = admin@grafana.localhost\n\n[emails]\n;welcome_email_on_sign_up = false\n\n#################################### Logging ##########################\n[log]\n# Either \"console\", \"file\", default is \"console\"\n# Use comma to separate multiple modes, e.g. \"console, file\"\n;mode = console, file\n\n# Buffer length of channel, keep it as it is if you don't know what it is.\n;buffer_len = 10000\n\n# Either \"Trace\", \"Debug\", \"Info\", \"Warn\", \"Error\", \"Critical\", default is \"Trace\"\n;level = Info\n\n# For \"console\" mode only\n[log.console]\n;level =\n\n# For \"file\" mode only\n[log.file]\n;level =\n# This enables automated log rotate(switch of following options), default is true\n;log_rotate = true\n\n# Max line number of single file, default is 1000000\n;max_lines = 1000000\n\n# Max size shift of single file, default is 28 means 1 << 28, 256MB\n;max_lines_shift = 28\n\n# Segment log daily, default is true\n;daily_rotate = true\n\n# Expired days of log file(delete after max days), default is 7\n;max_days = 7\n\n#################################### AMPQ Event Publisher ##########################\n[event_publisher]\n;enabled = false\n;rabbitmq_url = amqp://localhost/\n;exchange = grafana_events\n\n;#################################### Dashboard JSON files ##########################\n[dashboards.json]\n;enabled = false\n;path = /var/lib/grafana/dashboards\npath = /usr/lib/ambari-metrics-grafana/public/dashboards"
  1466. }
  1467. }
  1468. },
  1469. {
  1470. "hadoop-env" : {
  1471. "properties_attributes" : { },
  1472. "properties" : {
  1473. "hadoop_heapsize" : "1024",
  1474. "proxyuser_group" : "users",
  1475. "hadoop_root_logger" : "INFO,RFA",
  1476. "dtnode_heapsize" : "1024m",
  1477. "namenode_backup_dir" : "/tmp/upgrades",
  1478. "hdfs_user" : "hdfs",
  1479. "hadoop_pid_dir_prefix" : "/var/run/hadoop",
  1480. "content" : "\n # Set Hadoop-specific environment variables here.\n\n # The only required environment variable is JAVA_HOME. All others are\n # optional. When running a distributed configuration it is best to\n # set JAVA_HOME in this file, so that it is correctly defined on\n # remote nodes.\n\n # The java implementation to use. Required.\n export JAVA_HOME={{java_home}}\n export HADOOP_HOME_WARN_SUPPRESS=1\n\n # Hadoop home directory\n export HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}\n\n # Hadoop Configuration Directory\n\n {# this is different for HDP1 #}\n # Path to jsvc required by secure HDP 2.0 datanode\n export JSVC_HOME={{jsvc_path}}\n\n\n # The maximum amount of heap to use, in MB. Default is 1000.\n export HADOOP_HEAPSIZE=\"{{hadoop_heapsize}}\"\n\n export HADOOP_NAMENODE_INIT_HEAPSIZE=\"-Xms{{namenode_heapsize}}\"\n\n # Extra Java runtime options. Empty by default.\n export HADOOP_OPTS=\"-Djava.net.preferIPv4Stack=true ${HADOOP_OPTS}\"\n\n # Command specific options appended to HADOOP_OPTS when specified\n HADOOP_JOBTRACKER_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{jtnode_opt_newsize}} -XX:MaxNewSize={{jtnode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xmx{{jtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dhadoop.mapreduce.jobsummary.logger=INFO,JSA ${HADOOP_JOBTRACKER_OPTS}\"\n\n HADOOP_TASKTRACKER_OPTS=\"-server -Xmx{{ttnode_heapsize}} -Dhadoop.security.logger=ERROR,console -Dmapred.audit.logger=ERROR,console ${HADOOP_TASKTRACKER_OPTS}\"\n\n {% if java_version < 8 %}\n SHARED_HADOOP_NAMENODE_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -XX:PermSize={{namenode_opt_permsize}} -XX:MaxPermSize={{namenode_opt_maxpermsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{namenode_heapsize}} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT\"\n export HADOOP_NAMENODE_OPTS=\"${SHARED_HADOOP_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\\\"/usr/hdp/current/hadoop-hdfs-namenode/bin/kill-name-node\\\" -Dorg.mortbay.jetty.Request.maxFormContentSize=-1 ${HADOOP_NAMENODE_OPTS}\"\n export HADOOP_DATANODE_OPTS=\"-server -XX:ParallelGCThreads=4 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -XX:PermSize=128m -XX:MaxPermSize=256m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{dtnode_heapsize}} -Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_DATANODE_OPTS} -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly\"\n\n export HADOOP_SECONDARYNAMENODE_OPTS=\"${SHARED_HADOOP_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\\\"/usr/hdp/current/hadoop-hdfs-secondarynamenode/bin/kill-secondary-name-node\\\" ${HADOOP_SECONDARYNAMENODE_OPTS}\"\n\n # The following applies to multiple commands (fs, dfs, fsck, distcp etc)\n export HADOOP_CLIENT_OPTS=\"-Xmx${HADOOP_HEAPSIZE}m -XX:MaxPermSize=512m $HADOOP_CLIENT_OPTS\"\n\n {% else %}\n SHARED_HADOOP_NAMENODE_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{namenode_heapsize}} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT\"\n export HADOOP_NAMENODE_OPTS=\"${SHARED_HADOOP_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\\\"/usr/hdp/current/hadoop-hdfs-namenode/bin/kill-name-node\\\" -Dorg.mortbay.jetty.Request.maxFormContentSize=-1 ${HADOOP_NAMENODE_OPTS}\"\n export HADOOP_DATANODE_OPTS=\"-server -XX:ParallelGCThreads=4 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{dtnode_heapsize}} -Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_DATANODE_OPTS} -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly\"\n\n export HADOOP_SECONDARYNAMENODE_OPTS=\"${SHARED_HADOOP_NAMENODE_OPTS} -XX:OnOutOfMemoryError=\\\"/usr/hdp/current/hadoop-hdfs-secondarynamenode/bin/kill-secondary-name-node\\\" ${HADOOP_SECONDARYNAMENODE_OPTS}\"\n\n # The following applies to multiple commands (fs, dfs, fsck, distcp etc)\n export HADOOP_CLIENT_OPTS=\"-Xmx${HADOOP_HEAPSIZE}m $HADOOP_CLIENT_OPTS\"\n {% endif %}\n\n HADOOP_NFS3_OPTS=\"-Xmx{{nfsgateway_heapsize}}m -Dhadoop.security.logger=ERROR,DRFAS ${HADOOP_NFS3_OPTS}\"\n HADOOP_BALANCER_OPTS=\"-server -Xmx{{hadoop_heapsize}}m ${HADOOP_BALANCER_OPTS}\"\n\n\n # On secure datanodes, user to run the datanode as after dropping privileges\n export HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER:-{{hadoop_secure_dn_user}}}\n\n # Extra ssh options. Empty by default.\n export HADOOP_SSH_OPTS=\"-o ConnectTimeout=5 -o SendEnv=HADOOP_CONF_DIR\"\n\n # Where log files are stored. $HADOOP_HOME/logs by default.\n export HADOOP_LOG_DIR={{hdfs_log_dir_prefix}}/$USER\n\n # History server logs\n export HADOOP_MAPRED_LOG_DIR={{mapred_log_dir_prefix}}/$USER\n\n # Where log files are stored in the secure data environment.\n export HADOOP_SECURE_DN_LOG_DIR={{hdfs_log_dir_prefix}}/$HADOOP_SECURE_DN_USER\n\n # File naming remote slave hosts. $HADOOP_HOME/conf/slaves by default.\n # export HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves\n\n # host:path where hadoop code should be rsync'd from. Unset by default.\n # export HADOOP_MASTER=master:/home/$USER/src/hadoop\n\n # Seconds to sleep between slave commands. Unset by default. This\n # can be useful in large clusters, where, e.g., slave rsyncs can\n # otherwise arrive faster than the master can service them.\n # export HADOOP_SLAVE_SLEEP=0.1\n\n # The directory where pid files are stored. /tmp by default.\n export HADOOP_PID_DIR={{hadoop_pid_dir_prefix}}/$USER\n export HADOOP_SECURE_DN_PID_DIR={{hadoop_pid_dir_prefix}}/$HADOOP_SECURE_DN_USER\n\n # History server pid\n export HADOOP_MAPRED_PID_DIR={{mapred_pid_dir_prefix}}/$USER\n\n YARN_RESOURCEMANAGER_OPTS=\"-Dyarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY\"\n\n # A string representing this instance of hadoop. $USER by default.\n export HADOOP_IDENT_STRING=$USER\n\n # The scheduling priority for daemon processes. See 'man nice'.\n\n # export HADOOP_NICENESS=10\n\n # Add database libraries\n JAVA_JDBC_LIBS=\"\"\n if [ -d \"/usr/share/java\" ]; then\n for jarFile in `ls /usr/share/java | grep -E \"(mysql|ojdbc|postgresql|sqljdbc)\" 2>/dev/null`\n do\n JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile\n done\n fi\n\n # Add libraries to the hadoop classpath - some may not need a colon as they already include it\n export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}\n\n # Setting path to hdfs command line\n export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}\n\n # Mostly required for hadoop 2.0\n export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:{{hadoop_lib_home}}/native/Linux-{{architecture}}-64\n\n export HADOOP_OPTS=\"-Dhdp.version=$HDP_VERSION $HADOOP_OPTS\"\n\n\n # Fix temporary bug, when ulimit from conf files is not picked up, without full relogin.\n # Makes sense to fix only when runing DN as root\n if [ \"$command\" == \"datanode\" ] && [ \"$EUID\" -eq 0 ] && [ -n \"$HADOOP_SECURE_DN_USER\" ]; then\n {% if is_datanode_max_locked_memory_set %}\n ulimit -l {{datanode_max_locked_memory}}\n {% endif %}\n ulimit -n {{hdfs_user_nofile_limit}}\n fi\n\n {% if hadoop_custom_extensions_enabled %}\n #Enable custom extensions\n export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:{{stack_root}}/current/ext/hadoop/*\n {% endif %}\n\n # Enable ACLs on zookeper znodes if required\n {% if hadoop_zkfc_opts is defined %}\n export HADOOP_ZKFC_OPTS=\"{{hadoop_zkfc_opts}} $HADOOP_ZKFC_OPTS\"\n {% endif %}",
  1481. "hdfs_log_dir_prefix" : "/var/log/hadoop",
  1482. "namenode_opt_newsize" : "128m",
  1483. "namenode_heapsize" : "1024m",
  1484. "hdfs_tmp_dir" : "/tmp",
  1485. "namenode_opt_maxpermsize" : "256m",
  1486. "nfsgateway_heapsize" : "1024",
  1487. "hdfs_user_nofile_limit" : "128000",
  1488. "keyserver_host" : " ",
  1489. "keyserver_port" : "",
  1490. "hdfs_user_nproc_limit" : "65536",
  1491. "namenode_opt_maxnewsize" : "128m",
  1492. "namenode_opt_permsize" : "128m"
  1493. }
  1494. }
  1495. },
  1496. {
  1497. "zookeeper-env" : {
  1498. "properties_attributes" : { },
  1499. "properties" : {
  1500. "zk_log_dir" : "/var/log/zookeeper",
  1501. "zk_user" : "zookeeper",
  1502. "zk_pid_dir" : "/var/run/zookeeper",
  1503. "content" : "\nexport JAVA_HOME={{java64_home}}\nexport ZOOKEEPER_HOME={{zk_home}}\nexport ZOO_LOG_DIR={{zk_log_dir}}\nexport ZOOPIDFILE={{zk_pid_file}}\nexport SERVER_JVMFLAGS={{zk_server_heapsize}}\nexport JAVA=$JAVA_HOME/bin/java\nexport CLASSPATH=$CLASSPATH:/usr/share/zookeeper/*\n\n{% if security_enabled %}\nexport SERVER_JVMFLAGS=\"$SERVER_JVMFLAGS -Djava.security.auth.login.config={{zk_server_jaas_file}}\"\nexport CLIENT_JVMFLAGS=\"$CLIENT_JVMFLAGS -Djava.security.auth.login.config={{zk_client_jaas_file}}\"\n{% endif %}"
  1504. }
  1505. }
  1506. },
  1507. {
  1508. "yarn-log4j" : {
  1509. "properties_attributes" : { },
  1510. "properties" : {
  1511. "yarn_rm_summary_log_max_backup_size" : "256",
  1512. "yarn_rm_summary_log_number_of_backup_files" : "20",
  1513. "content" : "\n#Relative to Yarn Log Dir Prefix\nyarn.log.dir=.\n#\n# Job Summary Appender\n#\n# Use following logger to send summary to separate file defined by\n# hadoop.mapreduce.jobsummary.log.file rolled daily:\n# hadoop.mapreduce.jobsummary.logger=INFO,JSA\n#\nhadoop.mapreduce.jobsummary.logger=${hadoop.root.logger}\nhadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log\nlog4j.appender.JSA=org.apache.log4j.DailyRollingFileAppender\n# Set the ResourceManager summary log filename\nyarn.server.resourcemanager.appsummary.log.file=hadoop-mapreduce.jobsummary.log\n# Set the ResourceManager summary log level and appender\nyarn.server.resourcemanager.appsummary.logger=${hadoop.root.logger}\n#yarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY\n\n# To enable AppSummaryLogging for the RM,\n# set yarn.server.resourcemanager.appsummary.logger to\n# LEVEL,RMSUMMARY in hadoop-env.sh\n\n# Appender for ResourceManager Application Summary Log\n# Requires the following properties to be set\n# - hadoop.log.dir (Hadoop Log directory)\n# - yarn.server.resourcemanager.appsummary.log.file (resource manager app summary log filename)\n# - yarn.server.resourcemanager.appsummary.logger (resource manager app summary log level and appender)\nlog4j.appender.RMSUMMARY=org.apache.log4j.RollingFileAppender\nlog4j.appender.RMSUMMARY.File=${yarn.log.dir}/${yarn.server.resourcemanager.appsummary.log.file}\nlog4j.appender.RMSUMMARY.MaxFileSize={{yarn_rm_summary_log_max_backup_size}}MB\nlog4j.appender.RMSUMMARY.MaxBackupIndex={{yarn_rm_summary_log_number_of_backup_files}}\nlog4j.appender.RMSUMMARY.layout=org.apache.log4j.PatternLayout\nlog4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\nlog4j.appender.JSA.layout=org.apache.log4j.PatternLayout\nlog4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n\nlog4j.appender.JSA.DatePattern=.yyyy-MM-dd\nlog4j.appender.JSA.layout=org.apache.log4j.PatternLayout\nlog4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=${yarn.server.resourcemanager.appsummary.logger}\nlog4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=false\n\n# Appender for viewing information for errors and warnings\nyarn.ewma.cleanupInterval=300\nyarn.ewma.messageAgeLimitSeconds=86400\nyarn.ewma.maxUniqueMessages=250\nlog4j.appender.EWMA=org.apache.hadoop.yarn.util.Log4jWarningErrorMetricsAppender\nlog4j.appender.EWMA.cleanupInterval=${yarn.ewma.cleanupInterval}\nlog4j.appender.EWMA.messageAgeLimitSeconds=${yarn.ewma.messageAgeLimitSeconds}\nlog4j.appender.EWMA.maxUniqueMessages=${yarn.ewma.maxUniqueMessages}\n\n# Audit logging for ResourceManager\nrm.audit.logger=${hadoop.root.logger}\nlog4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger=${rm.audit.logger}\nlog4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger=false\nlog4j.appender.RMAUDIT=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.RMAUDIT.File=${yarn.log.dir}/rm-audit.log\nlog4j.appender.RMAUDIT.layout=org.apache.log4j.PatternLayout\nlog4j.appender.RMAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\nlog4j.appender.RMAUDIT.DatePattern=.yyyy-MM-dd\n\n# Audit logging for NodeManager\nnm.audit.logger=${hadoop.root.logger}\nlog4j.logger.org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger=${nm.audit.logger}\nlog4j.additivity.org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger=false\nlog4j.appender.NMAUDIT=org.apache.log4j.DailyRollingFileAppender\nlog4j.appender.NMAUDIT.File=${yarn.log.dir}/nm-audit.log\nlog4j.appender.NMAUDIT.layout=org.apache.log4j.PatternLayout\nlog4j.appender.NMAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n\nlog4j.appender.NMAUDIT.DatePattern=.yyyy-MM-dd"
  1514. }
  1515. }
  1516. },
  1517. {
  1518. "mapred-env" : {
  1519. "properties_attributes" : { },
  1520. "properties" : {
  1521. "jobhistory_heapsize" : "900",
  1522. "mapred_user_nofile_limit" : "32768",
  1523. "mapred_user" : "mapred",
  1524. "mapred_user_nproc_limit" : "65536",
  1525. "mapred_pid_dir_prefix" : "/var/run/hadoop-mapreduce",
  1526. "mapred_log_dir_prefix" : "/var/log/hadoop-mapreduce",
  1527. "content" : "\n# export JAVA_HOME=/home/y/libexec/jdk1.6.0/\n\nexport HADOOP_JOB_HISTORYSERVER_HEAPSIZE={{jobhistory_heapsize}}\n\nexport HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA\n\n{% if security_enabled %}\nexport HADOOP_JOB_HISTORYSERVER_OPTS=\"-Djava.security.auth.login.config={{mapred_jaas_file}} -Djavax.security.auth.useSubjectCredsOnly=false\"\n{% endif %}\n#export HADOOP_MAPRED_LOG_DIR=\"\" # Where log files are stored. $HADOOP_MAPRED_HOME/logs by default.\n#export HADOOP_JHS_LOGGER=INFO,RFA # Hadoop JobSummary logger.\n#export HADOOP_MAPRED_PID_DIR= # The pid files are stored. /tmp by default.\n#export HADOOP_MAPRED_IDENT_STRING= #A string representing this instance of hadoop. $USER by default\n#export HADOOP_MAPRED_NICENESS= #The scheduling priority for daemons. Defaults to 0.\nexport HADOOP_OPTS=\"-Dhdp.version=$HDP_VERSION $HADOOP_OPTS\"\nexport HADOOP_OPTS=\"-Djava.io.tmpdir={{hadoop_java_io_tmpdir}} $HADOOP_OPTS\"\nexport JAVA_LIBRARY_PATH=\"${JAVA_LIBRARY_PATH}:{{hadoop_java_io_tmpdir}}\""
  1528. }
  1529. }
  1530. },
  1531. {
  1532. "activity-zeppelin-interpreter" : {
  1533. "properties_attributes" : { },
  1534. "properties" : {
  1535. "activity-zeppelin-interpreter-content" : "{\n \"interpreterSettings\": {\n \"2BJB693M8\": {\n \"id\": \"2BJB693M8\",\n \"name\": \"phoenix\",\n \"group\": \"phoenix\",\n \"properties\": {\n \"phoenix.jdbc.url\": \"{{activity_explorer_jdbc_url}}\",\n \"phoenix.user\": \"\",\n \"phoenix.password\": \"\",\n \"phoenix.max.result\": \"1000\",\n \"phoenix.driver.name\": \"org.apache.phoenix.jdbc.PhoenixDriver\"\n },\n \"interpreterGroup\": [\n {\n \"class\": \"org.apache.zeppelin.phoenix.PhoenixInterpreter\",\n \"name\": \"sql\"\n }\n ],\n \"dependencies\": [],\n \"option\": {\n \"remote\": true,\n \"perNoteSession\": false\n }\n }\n },\n\"interpreterBindings\": {\n \"2BNVQJUBK\": [\n \"2BJB693M8\"\n ],\n \"2BPD7951H\": [\n \"2BJB693M8\"\n ],\n \"2BQH91X36\": [\n \"2BJB693M8\"\n ],\n \"2BTCVPTMH\": [\n \"2BJB693M8\"\n ]\n },\n\"interpreterRepositories\": [\n {\n \"id\": \"central\",\n \"type\": \"default\",\n \"url\": \"http://repo1.maven.org/maven2/\",\n \"releasePolicy\": {\n \"enabled\": true,\n \"updatePolicy\": \"daily\",\n \"checksumPolicy\": \"warn\"\n },\n \"snapshotPolicy\": {\n \"enabled\": true,\n \"updatePolicy\": \"daily\",\n \"checksumPolicy\": \"warn\"\n },\n \"mirroredRepositories\": [],\n \"repositoryManager\": false\n }\n ]\n}"
  1536. }
  1537. }
  1538. },
  1539. {
  1540. "cluster-env" : {
  1541. "properties_attributes" : { },
  1542. "properties" : {
  1543. "hide_yarn_memory_widget" : "false",
  1544. "enable_external_ranger" : "false",
  1545. "recovery_max_count" : "6",
  1546. "manage_dirs_on_root" : "true",
  1547. "managed_hdfs_resource_property_names" : "",
  1548. "recovery_retry_interval" : "5",
  1549. "stack_features" : "{\n \"HDP\": {\n \"stack_features\": [\n {\n \"name\": \"snappy\",\n \"description\": \"Snappy compressor/decompressor support\",\n \"min_version\": \"2.0.0.0\",\n \"max_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"lzo\",\n \"description\": \"LZO libraries support\",\n \"min_version\": \"2.2.1.0\"\n },\n {\n \"name\": \"express_upgrade\",\n \"description\": \"Express upgrade support\",\n \"min_version\": \"2.1.0.0\"\n },\n {\n \"name\": \"rolling_upgrade\",\n \"description\": \"Rolling upgrade support\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"kafka_acl_migration_support\",\n \"description\": \"ACL migration support\",\n \"min_version\": \"2.3.4.0\"\n },\n {\n \"name\": \"secure_zookeeper\",\n \"description\": \"Protect ZNodes with SASL acl in secure clusters\",\n \"min_version\": \"2.6.0.0\"\n },\n {\n \"name\": \"config_versioning\",\n \"description\": \"Configurable versions support\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"datanode_non_root\",\n \"description\": \"DataNode running as non-root support (AMBARI-7615)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"remove_ranger_hdfs_plugin_env\",\n \"description\": \"HDFS removes Ranger env files (AMBARI-14299)\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"ranger\",\n \"description\": \"Ranger Service support\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"ranger_tagsync_component\",\n \"description\": \"Ranger Tagsync component support (AMBARI-14383)\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"phoenix\",\n \"description\": \"Phoenix Service support\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"nfs\",\n \"description\": \"NFS support\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"tez_for_spark\",\n \"description\": \"Tez dependency for Spark\",\n \"min_version\": \"2.2.0.0\",\n \"max_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"timeline_state_store\",\n \"description\": \"Yarn application timeline-service supports state store property (AMBARI-11442)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"copy_tarball_to_hdfs\",\n \"description\": \"Copy tarball to HDFS support (AMBARI-12113)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"spark_16plus\",\n \"description\": \"Spark 1.6+\",\n \"min_version\": \"2.4.0.0\"\n },\n {\n \"name\": \"spark_thriftserver\",\n \"description\": \"Spark Thrift Server\",\n \"min_version\": \"2.3.2.0\"\n },\n {\n \"name\": \"storm_kerberos\",\n \"description\": \"Storm Kerberos support (AMBARI-7570)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"storm_ams\",\n \"description\": \"Storm AMS integration (AMBARI-10710)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"create_kafka_broker_id\",\n \"description\": \"Ambari should create Kafka Broker Id (AMBARI-12678)\",\n \"min_version\": \"2.2.0.0\",\n \"max_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"kafka_listeners\",\n \"description\": \"Kafka listeners (AMBARI-10984)\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"kafka_kerberos\",\n \"description\": \"Kafka Kerberos support (AMBARI-10984)\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"pig_on_tez\",\n \"description\": \"Pig on Tez support (AMBARI-7863)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"ranger_usersync_non_root\",\n \"description\": \"Ranger Usersync as non-root user (AMBARI-10416)\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"ranger_audit_db_support\",\n \"description\": \"Ranger Audit to DB support\",\n \"min_version\": \"2.2.0.0\",\n \"max_version\": \"2.4.99.99\"\n },\n {\n \"name\": \"accumulo_kerberos_user_auth\",\n \"description\": \"Accumulo Kerberos User Auth (AMBARI-10163)\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"knox_versioned_data_dir\",\n \"description\": \"Use versioned data dir for Knox (AMBARI-13164)\",\n \"min_version\": \"2.3.2.0\"\n },\n {\n \"name\": \"knox_sso_topology\",\n \"description\": \"Knox SSO Topology support (AMBARI-13975)\",\n \"min_version\": \"2.3.8.0\"\n },\n {\n \"name\": \"atlas_rolling_upgrade\",\n \"description\": \"Rolling upgrade support for Atlas\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"oozie_admin_user\",\n \"description\": \"Oozie install user as an Oozie admin user (AMBARI-7976)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"oozie_create_hive_tez_configs\",\n \"description\": \"Oozie create configs for Ambari Hive and Tez deployments (AMBARI-8074)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"oozie_setup_shared_lib\",\n \"description\": \"Oozie setup tools used to shared Oozie lib to HDFS (AMBARI-7240)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"oozie_host_kerberos\",\n \"description\": \"Oozie in secured clusters uses _HOST in Kerberos principal (AMBARI-9775)\",\n \"min_version\": \"2.0.0.0\"\n },\n {\n \"name\": \"falcon_extensions\",\n \"description\": \"Falcon Extension\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"hive_metastore_upgrade_schema\",\n \"description\": \"Hive metastore upgrade schema support (AMBARI-11176)\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"hive_server_interactive\",\n \"description\": \"Hive server interactive support (AMBARI-15573)\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"hive_webhcat_specific_configs\",\n \"description\": \"Hive webhcat specific configurations support (AMBARI-12364)\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"hive_purge_table\",\n \"description\": \"Hive purge table support (AMBARI-12260)\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"hive_server2_kerberized_env\",\n \"description\": \"Hive server2 working on kerberized environment (AMBARI-13749)\",\n \"min_version\": \"2.2.3.0\",\n \"max_version\": \"2.2.5.0\"\n },\n {\n \"name\": \"hive_env_heapsize\",\n \"description\": \"Hive heapsize property defined in hive-env (AMBARI-12801)\",\n \"min_version\": \"2.2.0.0\"\n },\n {\n \"name\": \"ranger_kms_hsm_support\",\n \"description\": \"Ranger KMS HSM support (AMBARI-15752)\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"ranger_log4j_support\",\n \"description\": \"Ranger supporting log-4j properties (AMBARI-15681)\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"ranger_kerberos_support\",\n \"description\": \"Ranger Kerberos support\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"hive_metastore_site_support\",\n \"description\": \"Hive Metastore site support\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"ranger_usersync_password_jceks\",\n \"description\": \"Saving Ranger Usersync credentials in jceks\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"ranger_install_infra_client\",\n \"description\": \"Ambari Infra Service support\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"falcon_atlas_support_2_3\",\n \"description\": \"Falcon Atlas integration support for 2.3 stack\",\n \"min_version\": \"2.3.99.0\",\n \"max_version\": \"2.4.0.0\"\n },\n {\n \"name\": \"falcon_atlas_support\",\n \"description\": \"Falcon Atlas integration\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"hbase_home_directory\",\n \"description\": \"Hbase home directory in HDFS needed for HBASE backup\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"spark_livy\",\n \"description\": \"Livy as slave component of spark\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"spark_livy2\",\n \"description\": \"Livy as slave component of spark\",\n \"min_version\": \"2.6.0.0\"\n },\n {\n \"name\": \"atlas_ranger_plugin_support\",\n \"description\": \"Atlas Ranger plugin support\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"atlas_conf_dir_in_path\",\n \"description\": \"Prepend the Atlas conf dir (/etc/atlas/conf) to the classpath of Storm and Falcon\",\n \"min_version\": \"2.3.0.0\",\n \"max_version\": \"2.4.99.99\"\n },\n {\n \"name\": \"atlas_upgrade_support\",\n \"description\": \"Atlas supports express and rolling upgrades\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"atlas_hook_support\",\n \"description\": \"Atlas support for hooks in Hive, Storm, Falcon, and Sqoop\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"ranger_pid_support\",\n \"description\": \"Ranger Service support pid generation AMBARI-16756\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"ranger_kms_pid_support\",\n \"description\": \"Ranger KMS Service support pid generation\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"ranger_admin_password_change\",\n \"description\": \"Allow ranger admin credentials to be specified during cluster creation (AMBARI-17000)\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"ranger_setup_db_on_start\",\n \"description\": \"Allows setup of ranger db and java patches to be called multiple times on each START\",\n \"min_version\": \"2.6.0.0\"\n },\n {\n \"name\": \"storm_metrics_apache_classes\",\n \"description\": \"Metrics sink for Storm that uses Apache class names\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"spark_java_opts_support\",\n \"description\": \"Allow Spark to generate java-opts file\",\n \"min_version\": \"2.2.0.0\",\n \"max_version\": \"2.4.0.0\"\n },\n {\n \"name\": \"atlas_hbase_setup\",\n \"description\": \"Use script to create Atlas tables in Hbase and set permissions for Atlas user.\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"ranger_hive_plugin_jdbc_url\",\n \"description\": \"Handle Ranger hive repo config jdbc url change for stack 2.5 (AMBARI-18386)\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"zkfc_version_advertised\",\n \"description\": \"ZKFC advertise version\",\n \"min_version\": \"2.5.0.0\"\n },\n {\n \"name\": \"phoenix_core_hdfs_site_required\",\n \"description\": \"HDFS and CORE site required for Phoenix\",\n \"max_version\": \"2.5.9.9\"\n },\n {\n \"name\": \"ranger_tagsync_ssl_xml_support\",\n \"description\": \"Ranger Tagsync ssl xml support.\",\n \"min_version\": \"2.6.0.0\"\n },\n {\n \"name\": \"ranger_xml_configuration\",\n \"description\": \"Ranger code base support xml configurations\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"kafka_ranger_plugin_support\",\n \"description\": \"Ambari stack changes for Ranger Kafka Plugin (AMBARI-11299)\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"yarn_ranger_plugin_support\",\n \"description\": \"Implement Stack changes for Ranger Yarn Plugin integration (AMBARI-10866)\",\n \"min_version\": \"2.3.0.0\"\n },\n {\n \"name\": \"ranger_solr_config_support\",\n \"description\": \"Showing Ranger solrconfig.xml on UI\",\n \"min_version\": \"2.6.0.0\"\n },\n {\n \"name\": \"hive_interactive_atlas_hook_required\",\n \"description\": \"Registering Atlas Hook for Hive Interactive.\",\n \"min_version\": \"2.6.0.0\"\n },\n {\n \"name\": \"core_site_for_ranger_plugins\",\n \"description\": \"Adding core-site.xml in when Ranger plugin is enabled for Storm, Kafka, and Knox.\",\n \"min_version\": \"2.6.0.0\"\n },\n {\n \"name\": \"atlas_install_hook_package_support\",\n \"description\": \"Stop installing packages from 2.6\",\n \"max_version\": \"2.5.9.9\"\n },\n {\n \"name\": \"atlas_hdfs_site_on_namenode_ha\",\n \"description\": \"Need to create hdfs-site under atlas-conf dir when Namenode-HA is enabled.\",\n \"min_version\": \"2.6.0.0\"\n },\n {\n \"name\": \"hive_interactive_ga\",\n \"description\": \"Hive Interactive GA support\",\n \"min_version\": \"2.6.0.0\"\n },\n {\n \"name\": \"secure_ranger_ssl_password\",\n \"description\": \"Securing Ranger Admin and Usersync SSL and Trustore related passwords in jceks\",\n \"min_version\": \"2.6.0.0\"\n },\n {\n \"name\": \"ranger_kms_ssl\",\n \"description\": \"Ranger KMS SSL properties in ambari stack\",\n \"min_version\": \"2.6.0.0\"\n },\n {\n \"name\": \"nifi_encrypt_config\",\n \"description\": \"Encrypt sensitive properties written to nifi property file\",\n \"min_version\": \"2.6.0.0\"\n },\n {\n \"name\": \"toolkit_config_update\",\n \"description\": \"Support separate input and output for toolkit configuration\",\n \"min_version\": \"2.6.0.0\"\n },\n {\n \"name\": \"admin_toolkit_support\",\n \"description\": \"Supports the nifi admin toolkit\",\n \"min_version\": \"2.6.0.0\"\n },\n {\n \"name\": \"tls_toolkit_san\",\n \"description\": \"Support subject alternative name flag\",\n \"min_version\": \"2.6.0.0\"\n },\n {\n \"name\": \"nifi_jaas_conf_create\",\n \"description\": \"Create NIFI jaas configuration when kerberos is enabled\",\n \"min_version\": \"2.6.0.0\"\n },\n {\n \"name\": \"atlas_core_site_support\",\n \"description\": \"Need to create core-site under Atlas conf directory.\",\n \"min_version\": \"2.6.0.0\"\n },\n {\n \"name\": \"hadoop_custom_extensions\",\n \"description\": \"Support hadoop custom extensions\",\n \"min_version\": \"2.6.0.0\"\n }\n ]\n }\n}",
  1550. "stack_tools" : "{\n \"HDP\": {\n \"stack_selector\": [\n \"hdp-select\",\n \"/usr/bin/hdp-select\",\n \"hdp-select\"\n ],\n \"conf_selector\": [\n \"conf-select\",\n \"/usr/bin/conf-select\",\n \"conf-select\"\n ]\n }\n}",
  1551. "repo_ubuntu_template" : "{{package_type}} {{base_url}} {{components}}",
  1552. "stack_root" : "{\"HDP\":\"/usr/hdp\"}",
  1553. "fetch_nonlocal_groups" : "true",
  1554. "one_dir_per_partition" : "false",
  1555. "kerberos_domain" : "EXAMPLE.COM",
  1556. "ignore_bad_mounts" : "false",
  1557. "user_group" : "hadoop",
  1558. "sysprep_skip_setup_jce" : "false",
  1559. "override_uid" : "true",
  1560. "security_enabled" : "false",
  1561. "sysprep_skip_copy_fast_jar_hdfs" : "false",
  1562. "agent_mounts_ignore_list" : "",
  1563. "recovery_lifetime_max_count" : "1024",
  1564. "recovery_type" : "AUTO_START",
  1565. "repo_suse_rhel_template" : "[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0",
  1566. "sysprep_skip_create_users_and_groups" : "false",
  1567. "sysprep_skip_copy_tarballs_hdfs" : "false",
  1568. "sysprep_skip_copy_oozie_share_lib_to_hdfs" : "false",
  1569. "recovery_enabled" : "true",
  1570. "smokeuser" : "ambari-qa",
  1571. "smokeuser_keytab" : "/etc/security/keytabs/smokeuser.headless.keytab",
  1572. "alerts_repeat_tolerance" : "1",
  1573. "recovery_window_in_minutes" : "60",
  1574. "stack_name" : "HDP",
  1575. "ignore_groupsusers_create" : "false"
  1576. }
  1577. }
  1578. },
  1579. {
  1580. "hive-env" : {
  1581. "properties_attributes" : { },
  1582. "properties" : {
  1583. "webhcat_user" : "hcat",
  1584. "hive_log_dir" : "/var/log/hive",
  1585. "hive_security_authorization" : "None",
  1586. "hive_user_nproc_limit" : "16000",
  1587. "content" : "\nexport HADOOP_USER_CLASSPATH_FIRST=true #this prevents old metrics libs from mapreduce lib from bringing in old jar deps overriding HIVE_LIB\nif [ \"$SERVICE\" = \"cli\" ]; then\n if [ -z \"$DEBUG\" ]; then\n export HADOOP_OPTS=\"$HADOOP_OPTS -XX:NewRatio=12 -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:+UseNUMA -XX:+UseParallelGC -XX:-UseGCOverheadLimit\"\n else\n export HADOOP_OPTS=\"$HADOOP_OPTS -XX:NewRatio=12 -XX:MaxHeapFreeRatio=40 -XX:MinHeapFreeRatio=15 -XX:-UseGCOverheadLimit\"\n fi\nfi\n\n# The heap size of the jvm stared by hive shell script can be controlled via:\n\nif [ \"$SERVICE\" = \"metastore\" ]; then\n export HADOOP_HEAPSIZE={{hive_metastore_heapsize}} # Setting for HiveMetastore\nelse\n export HADOOP_HEAPSIZE={{hive_heapsize}} # Setting for HiveServer2 and Client\nfi\n\nexport HADOOP_CLIENT_OPTS=\"$HADOOP_CLIENT_OPTS -Xmx${HADOOP_HEAPSIZE}m\"\nexport HADOOP_CLIENT_OPTS=\"$HADOOP_CLIENT_OPTS{{heap_dump_opts}}\"\n\n# Larger heap size may be required when running queries over large number of files or partitions.\n# By default hive shell scripts use a heap size of 256 (MB). Larger heap size would also be\n# appropriate for hive server (hwi etc).\n\n\n# Set HADOOP_HOME to point to a specific hadoop install directory\nHADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}\n\nexport HIVE_HOME=${HIVE_HOME:-{{hive_home_dir}}}\n\n# Hive Configuration Directory can be controlled by:\nexport HIVE_CONF_DIR=${HIVE_CONF_DIR:-{{hive_config_dir}}}\n\n# Folder containing extra libraries required for hive compilation/execution can be controlled by:\nexport HIVE_AUX_JARS_PATH={{stack_root}}/current/ext/hive\nif [ \"${HIVE_AUX_JARS_PATH}\" != \"\" ]; then\n if [ -f \"${HIVE_AUX_JARS_PATH}\" ] || [ -d \"${HIVE_AUX_JARS_PATH}\" ] ; then\n export HIVE_AUX_JARS_PATH=${HIVE_AUX_JARS_PATH}\n elif [ -d \"/usr/hdp/current/hive-webhcat/share/hcatalog\" ]; then\n export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-webhcat/share/hcatalog/hive-hcatalog-core.jar\n fi\nelif [ -d \"/usr/hdp/current/hive-webhcat/share/hcatalog\" ]; then\n export HIVE_AUX_JARS_PATH=/usr/hdp/current/hive-webhcat/share/hcatalog/hive-hcatalog-core.jar\nfi\n\nexport METASTORE_PORT={{hive_metastore_port}}\n\n{% if sqla_db_used or lib_dir_available %}\nexport LD_LIBRARY_PATH=\"$LD_LIBRARY_PATH:{{jdbc_libs_dir}}\"\nexport JAVA_LIBRARY_PATH=\"$JAVA_LIBRARY_PATH:{{jdbc_libs_dir}}\"\n{% endif %}",
  1588. "hive.atlas.hook" : "false",
  1589. "hive_txn_acid" : "off",
  1590. "hive.client.heapsize" : "1024",
  1591. "hive.heapsize" : "3741",
  1592. "hive_database_type" : "mysql",
  1593. "hive_user_nofile_limit" : "32000",
  1594. "alert_ldap_username" : "",
  1595. "hcat_pid_dir" : "/var/run/webhcat",
  1596. "heap_dump_location" : "/tmp",
  1597. "hive_database" : "New MySQL Database",
  1598. "hcat_log_dir" : "/var/log/webhcat",
  1599. "hive_exec_orc_storage_strategy" : "SPEED",
  1600. "hive_user" : "hive",
  1601. "hive.metastore.heapsize" : "1247",
  1602. "hive_timeline_logging_enabled" : "true",
  1603. "hive.log.level" : "INFO",
  1604. "hive_database_name" : "hive",
  1605. "enable_heap_dump" : "false",
  1606. "hive_pid_dir" : "/var/run/hive",
  1607. "hcat_user" : "hcat",
  1608. "hive_ambari_database" : "MySQL"
  1609. }
  1610. }
  1611. },
  1612. {
  1613. "ams-grafana-env" : {
  1614. "properties_attributes" : { },
  1615. "properties" : {
  1616. "metrics_grafana_log_dir" : "/var/log/ambari-metrics-grafana",
  1617. "metrics_grafana_pid_dir" : "/var/run/ambari-metrics-grafana",
  1618. "content" : "\n# Set environment variables here.\n\n# AMS UI Server Home Dir\nexport AMS_GRAFANA_HOME_DIR={{ams_grafana_home_dir}}\n\n# AMS UI Server Data Dir\nexport AMS_GRAFANA_DATA_DIR={{ams_grafana_data_dir}}\n\n# AMS UI Server Log Dir\nexport AMS_GRAFANA_LOG_DIR={{ams_grafana_log_dir}}\n\n# AMS UI Server PID Dir\nexport AMS_GRAFANA_PID_DIR={{ams_grafana_pid_dir}}",
  1619. "metrics_grafana_data_dir" : "/var/lib/ambari-metrics-grafana",
  1620. "metrics_grafana_username" : "admin"
  1621. }
  1622. }
  1623. },
  1624. {
  1625. "hive-interactive-site" : {
  1626. "properties_attributes" : { },
  1627. "properties" : {
  1628. "hive.vectorized.groupby.maxentries" : "1000000",
  1629. "hive.llap.client.consistent.splits" : "true",
  1630. "hive.llap.daemon.yarn.container.mb" : "0",
  1631. "hive.llap.io.memory.size" : "0",
  1632. "hive.llap.daemon.am.liveness.heartbeat.interval.ms" : "10000ms",
  1633. "hive.llap.task.scheduler.locality.delay" : "-1",
  1634. "hive.vectorized.execution.reduce.enabled" : "true",
  1635. "hive.llap.daemon.rpc.port" : "0",
  1636. "hive.execution.mode" : "llap",
  1637. "hive.exec.orc.split.strategy" : "HYBRID",
  1638. "hive.llap.io.enabled" : "true",
  1639. "hive.vectorized.execution.mapjoin.native.fast.hashtable.enabled" : "true",
  1640. "hive.optimize.dynamic.partition.hashjoin" : "true",
  1641. "hive.mapjoin.hybridgrace.hashtable" : "false",
  1642. "hive.execution.engine" : "tez",
  1643. "hive.llap.object.cache.enabled" : "true",
  1644. "hive.llap.daemon.queue.name" : "default",
  1645. "hive.tez.container.size" : "682",
  1646. "hive.metastore.uris" : "",
  1647. "hive.server2.tez.sessions.restricted.configs" : "hive.execution.mode,hive.execution.engine",
  1648. "hive.tez.cartesian-product.enabled" : "true",
  1649. "hive.llap.auto.allow.uber" : "false",
  1650. "hive.server2.webui.port" : "10502",
  1651. "hive.llap.daemon.num.executors" : "1",
  1652. "hive.llap.daemon.task.scheduler.enable.preemption" : "true",
  1653. "hive.llap.io.use.lrfu" : "true",
  1654. "hive.vectorized.execution.mapjoin.native.enabled" : "true",
  1655. "hive.metastore.event.listeners" : "",
  1656. "hive.server2.tez.default.queues" : "default",
  1657. "hive.llap.zk.sm.connectionString" : "%HOSTGROUP::host_group_1%:2181",
  1658. "hive.server2.tez.sessions.custom.queue.allowed" : "ignore",
  1659. "hive.server2.tez.sessions.per.default.queue" : "1",
  1660. "hive.server2.webui.use.ssl" : "false",
  1661. "hive.llap.management.rpc.port" : "15004",
  1662. "hive.server2.thrift.http.port" : "10501",
  1663. "hive.map.aggr.hash.min.reduction" : "0.99",
  1664. "hive.merge.nway.joins" : "false",
  1665. "hive.llap.daemon.logger" : "query-routing",
  1666. "hive.prewarm.enabled" : "false",
  1667. "hive.auto.convert.join.noconditionaltask.size" : "1000000000",
  1668. "llap.shuffle.connection-keep-alive.timeout" : "60",
  1669. "hive.llap.execution.mode" : "only",
  1670. "hive.llap.daemon.yarn.shuffle.port" : "15551",
  1671. "hive.llap.enable.grace.join.in.llap" : "false",
  1672. "hive.llap.daemon.vcpus.per.instance" : "${hive.llap.daemon.num.executors}",
  1673. "hive.llap.io.memory.mode" : "",
  1674. "hive.server2.thrift.port" : "10500",
  1675. "hive.driver.parallel.compilation" : "true",
  1676. "hive.tez.exec.print.summary" : "true",
  1677. "hive.tez.input.generate.consistent.splits" : "true",
  1678. "hive.vectorized.execution.mapjoin.minmax.enabled" : "true",
  1679. "hive.server2.enable.doAs" : "false",
  1680. "hive.server2.zookeeper.namespace" : "hiveserver2-hive2",
  1681. "hive.tez.bucket.pruning" : "true",
  1682. "hive.llap.io.threadpool.size" : "2",
  1683. "llap.shuffle.connection-keep-alive.enable" : "true",
  1684. "hive.llap.daemon.service.hosts" : "@llap0",
  1685. "hive.server2.tez.initialize.default.sessions" : "true"
  1686. }
  1687. }
  1688. },
  1689. {
  1690. "ranger-hive-audit" : {
  1691. "properties_attributes" : { },
  1692. "properties" : { }
  1693. }
  1694. },
  1695. {
  1696. "ranger-hive-security" : {
  1697. "properties_attributes" : { },
  1698. "properties" : { }
  1699. }
  1700. },
  1701. {
  1702. "slider-log4j" : {
  1703. "properties_attributes" : { },
  1704. "properties" : {
  1705. "content" : "\n# Licensed to the Apache Software Foundation (ASF) under one\n# or more contributor license agreements. See the NOTICE file\n# distributed with this work for additional information\n# regarding copyright ownership. The ASF licenses this file\n# to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file except in compliance\n# with the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n\n# Define some default values that can be overridden by system properties\nlog4j.rootLogger=INFO,stdout\nlog4j.threshhold=ALL\nlog4j.appender.stdout=org.apache.log4j.ConsoleAppender\nlog4j.appender.stdout.layout=org.apache.log4j.PatternLayout\n\n# log layout skips stack-trace creation operations by avoiding line numbers and method\nlog4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - %m%n\n\n# debug edition is much more expensive\n#log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n\n\n\nlog4j.appender.subprocess=org.apache.log4j.ConsoleAppender\nlog4j.appender.subprocess.layout=org.apache.log4j.PatternLayout\nlog4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n\n#log4j.logger.org.apache.slider.yarn.appmaster.SliderAppMasterer.master=INFO,subprocess\n\n# for debugging Slider\n#log4j.logger.org.apache.slider=DEBUG\n#log4j.logger.org.apache.slider=DEBUG\n\n# uncomment to debug service lifecycle issues\n#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG\n#log4j.logger.org.apache.hadoop.yarn.service=DEBUG\n\n# uncomment for YARN operations\n#log4j.logger.org.apache.hadoop.yarn.client=DEBUG\n\n# uncomment this to debug security problems\n#log4j.logger.org.apache.hadoop.security=DEBUG\n\n#crank back on some noise\nlog4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR\nlog4j.logger.org.apache.hadoop.hdfs=WARN\n\n\nlog4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN\nlog4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl=WARN\nlog4j.logger.org.apache.zookeeper=WARN"
  1706. }
  1707. }
  1708. },
  1709. {
  1710. "product-info" : {
  1711. "properties_attributes" : { },
  1712. "properties" : {
  1713. "product-info-content" : "\n{\n \"schemaVersion\" : \"1.0.0\",\n \"productId\": \"{{stackName}}\",\n \"componentId\": \"{{stackName}}\",\n \"productVersion\" : \"{{stackVersion}}\",\n \"type\":\"cluster\",\n \"instanceInfo\": {\n \"guid\": \"\",\n \"parentGuid\": \"\",\n \"name\":\"{{clusterName}}\",\n \"flexSubscriptionId\": \"{{flexSubscriptionId}}\",\n \"provider\": \"\",\n \"region\": \"\"\n }\n}"
  1714. }
  1715. }
  1716. }
  1717. ],
  1718. "host_groups" : [
  1719. {
  1720. "components" : [
  1721. {
  1722. "name" : "NODEMANAGER"
  1723. },
  1724. {
  1725. "name" : "HIVE_SERVER"
  1726. },
  1727. {
  1728. "name" : "METRICS_MONITOR"
  1729. },
  1730. {
  1731. "name" : "NFS_GATEWAY"
  1732. },
  1733. {
  1734. "name" : "HIVE_METASTORE"
  1735. },
  1736. {
  1737. "name" : "TEZ_CLIENT"
  1738. },
  1739. {
  1740. "name" : "ZOOKEEPER_CLIENT"
  1741. },
  1742. {
  1743. "name" : "HCAT"
  1744. },
  1745. {
  1746. "name" : "WEBHCAT_SERVER"
  1747. },
  1748. {
  1749. "name" : "ACTIVITY_ANALYZER"
  1750. },
  1751. {
  1752. "name" : "SECONDARY_NAMENODE"
  1753. },
  1754. {
  1755. "name" : "HST_AGENT"
  1756. },
  1757. {
  1758. "name" : "SLIDER"
  1759. },
  1760. {
  1761. "name" : "ZOOKEEPER_SERVER"
  1762. },
  1763. {
  1764. "name" : "METRICS_COLLECTOR"
  1765. },
  1766. {
  1767. "name" : "METRICS_GRAFANA"
  1768. },
  1769. {
  1770. "name" : "YARN_CLIENT"
  1771. },
  1772. {
  1773. "name" : "HDFS_CLIENT"
  1774. },
  1775. {
  1776. "name" : "HST_SERVER"
  1777. },
  1778. {
  1779. "name" : "MYSQL_SERVER"
  1780. },
  1781. {
  1782. "name" : "HISTORYSERVER"
  1783. },
  1784. {
  1785. "name" : "NAMENODE"
  1786. },
  1787. {
  1788. "name" : "PIG"
  1789. },
  1790. {
  1791. "name" : "ACTIVITY_EXPLORER"
  1792. },
  1793. {
  1794. "name" : "MAPREDUCE2_CLIENT"
  1795. },
  1796. {
  1797. "name" : "AMBARI_SERVER"
  1798. },
  1799. {
  1800. "name" : "DATANODE"
  1801. },
  1802. {
  1803. "name" : "APP_TIMELINE_SERVER"
  1804. },
  1805. {
  1806. "name" : "HIVE_CLIENT"
  1807. },
  1808. {
  1809. "name" : "RESOURCEMANAGER"
  1810. }
  1811. ],
  1812. "configurations" : [ ],
  1813. "name" : "host_group_1",
  1814. "cardinality" : "1"
  1815. }
  1816. ],
  1817. "settings" : [
  1818. {
  1819. "recovery_settings" : [
  1820. {
  1821. "recovery_enabled" : "true"
  1822. }
  1823. ]
  1824. },
  1825. {
  1826. "service_settings" : [
  1827. {
  1828. "name" : "HIVE",
  1829. "credential_store_enabled" : "true"
  1830. },
  1831. {
  1832. "recovery_enabled" : "true",
  1833. "name" : "AMBARI_METRICS"
  1834. }
  1835. ]
  1836. },
  1837. {
  1838. "component_settings" : [
  1839. {
  1840. "recovery_enabled" : "true",
  1841. "name" : "METRICS_COLLECTOR"
  1842. }
  1843. ]
  1844. }
  1845. ],
  1846. "Blueprints" : {
  1847. "stack_name" : "HDP",
  1848. "stack_version" : "2.6"
  1849. }
  1850. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!