Advertisement
Guest User

Untitled

a guest
May 30th, 2016
66
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 39.92 KB | None | 0 0
  1. <?xml version="1.0" encoding="UTF-8" standalone="yes"?><template><description></description><name>TestConvertAvroToOrc2</name><snippet><connections><id>02fe77fc-2c72-461d-ba76-cd7ccfe4c0ec</id><parentGroupId>9007185c-8178-4606-81a5-b6a4e930ec8b</parentGroupId><backPressureDataSizeThreshold>0 MB</backPressureDataSizeThreshold><backPressureObjectThreshold>0</backPressureObjectThreshold><destination><groupId>9007185c-8178-4606-81a5-b6a4e930ec8b</groupId><id>17ad09e6-4c2e-4740-a046-f02433fbedc9</id><type>PROCESSOR</type></destination><flowFileExpiration>0 sec</flowFileExpiration><labelIndex>1</labelIndex><name></name><selectedRelationships>success</selectedRelationships><source><groupId>9007185c-8178-4606-81a5-b6a4e930ec8b</groupId><id>4a2f1555-737d-4d30-a717-75f8cbdb4d61</id><type>PROCESSOR</type></source><zIndex>0</zIndex></connections><connections><id>eba4991d-881b-4c1c-a4bc-5cc25b02e79d</id><parentGroupId>9007185c-8178-4606-81a5-b6a4e930ec8b</parentGroupId><backPressureDataSizeThreshold>0 MB</backPressureDataSizeThreshold><backPressureObjectThreshold>0</backPressureObjectThreshold><destination><groupId>9007185c-8178-4606-81a5-b6a4e930ec8b</groupId><id>e8e81b14-8c6b-4ac2-beab-31b4ed131d9a</id><type>PROCESSOR</type></destination><flowFileExpiration>0 sec</flowFileExpiration><labelIndex>1</labelIndex><name></name><selectedRelationships>success</selectedRelationships><source><groupId>9007185c-8178-4606-81a5-b6a4e930ec8b</groupId><id>658c8290-dd14-4968-9cc4-48c7d4685106</id><type>PROCESSOR</type></source><zIndex>0</zIndex></connections><connections><id>747ad317-00f7-46d7-9545-0df03e811622</id><parentGroupId>9007185c-8178-4606-81a5-b6a4e930ec8b</parentGroupId><backPressureDataSizeThreshold>0 MB</backPressureDataSizeThreshold><backPressureObjectThreshold>0</backPressureObjectThreshold><destination><groupId>9007185c-8178-4606-81a5-b6a4e930ec8b</groupId><id>658c8290-dd14-4968-9cc4-48c7d4685106</id><type>PROCESSOR</type></destination><flowFileExpiration>0 sec</flowFileExpiration><labelIndex>1</labelIndex><name></name><selectedRelationships>success</selectedRelationships><source><groupId>9007185c-8178-4606-81a5-b6a4e930ec8b</groupId><id>17ad09e6-4c2e-4740-a046-f02433fbedc9</id><type>PROCESSOR</type></source><zIndex>0</zIndex></connections><connections><id>d88e017e-f29c-4479-8343-6d3286cb6be9</id><parentGroupId>9007185c-8178-4606-81a5-b6a4e930ec8b</parentGroupId><backPressureDataSizeThreshold>0 MB</backPressureDataSizeThreshold><backPressureObjectThreshold>0</backPressureObjectThreshold><destination><groupId>9007185c-8178-4606-81a5-b6a4e930ec8b</groupId><id>fea02638-ee1c-4419-a9b7-5e6b1ab57380</id><type>PROCESSOR</type></destination><flowFileExpiration>0 sec</flowFileExpiration><labelIndex>1</labelIndex><name></name><selectedRelationships>success</selectedRelationships><source><groupId>9007185c-8178-4606-81a5-b6a4e930ec8b</groupId><id>e8e81b14-8c6b-4ac2-beab-31b4ed131d9a</id><type>PROCESSOR</type></source><zIndex>0</zIndex></connections><connections><id>364fa74a-f911-4d2a-861c-5a823761ea81</id><parentGroupId>9007185c-8178-4606-81a5-b6a4e930ec8b</parentGroupId><backPressureDataSizeThreshold>0 MB</backPressureDataSizeThreshold><backPressureObjectThreshold>0</backPressureObjectThreshold><destination><groupId>9007185c-8178-4606-81a5-b6a4e930ec8b</groupId><id>1b46938d-13fb-4e05-8334-e57f5d76edec</id><type>PROCESSOR</type></destination><flowFileExpiration>0 sec</flowFileExpiration><labelIndex>1</labelIndex><name></name><selectedRelationships>success</selectedRelationships><source><groupId>9007185c-8178-4606-81a5-b6a4e930ec8b</groupId><id>fea02638-ee1c-4419-a9b7-5e6b1ab57380</id><type>PROCESSOR</type></source><zIndex>0</zIndex></connections><processors><id>658c8290-dd14-4968-9cc4-48c7d4685106</id><parentGroupId>9007185c-8178-4606-81a5-b6a4e930ec8b</parentGroupId><position><x>-3.5484466552734375</x><y>249.89371299743652</y></position><config><bulletinLevel>WARN</bulletinLevel><comments></comments><concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount><defaultConcurrentTasks><entry><key>TIMER_DRIVEN</key><value>1</value></entry><entry><key>EVENT_DRIVEN</key><value>0</value></entry><entry><key>CRON_DRIVEN</key><value>1</value></entry></defaultConcurrentTasks><defaultSchedulingPeriod><entry><key>TIMER_DRIVEN</key><value>0 sec</value></entry><entry><key>CRON_DRIVEN</key><value>* * * * * ?</value></entry></defaultSchedulingPeriod><descriptors><entry><key>orc-config-resources</key><value><description>A file or comma separated list of files which contains the ORC configuration (hive-site.xml, e.g.). Without this, Hadoop will search the classpath for a 'hive-site.xml' file or will revert to a default configuration. Please see the ORC documentation for more details.</description><displayName>ORC Configuration Resources</displayName><dynamic>false</dynamic><name>orc-config-resources</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>orc-stripe-size</key><value><defaultValue>100000</defaultValue><description>The size of the memory buffer (in bytes) for writing stripes to an ORC file</description><displayName>Stripe Size</displayName><dynamic>false</dynamic><name>orc-stripe-size</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>orc-buffer-size</key><value><defaultValue>10000</defaultValue><description>The maximum size of the memory buffers (in bytes) used for compressing and storing a stripe in memory. This is a hint to the ORC writer, which may choose to use a smaller buffer size based on stripe size and number of columns for efficient stripe writing and memory utilization.</description><displayName>Buffer Size</displayName><dynamic>false</dynamic><name>orc-buffer-size</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>orc-compression-type</key><value><allowableValues><displayName>NONE</displayName><value>NONE</value></allowableValues><allowableValues><displayName>ZLIB</displayName><value>ZLIB</value></allowableValues><allowableValues><displayName>SNAPPY</displayName><value>SNAPPY</value></allowableValues><allowableValues><displayName>LZO</displayName><value>LZO</value></allowableValues><defaultValue>NONE</defaultValue><description></description><displayName>Compression Type</displayName><dynamic>false</dynamic><name>orc-compression-type</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>orc-hive-table-name</key><value><description>An optional table name to insert into the hive.ddl attribute. The generated DDL can be used by a PutHiveQL processor (presumably after a PutHDFS processor) to create a table backed by the converted ORC file. If this property is not provided, the full name (including namespace) of the incoming Avro record will be normalized and used as the table name.</description><displayName>Hive Table Name</displayName><dynamic>false</dynamic><name>orc-hive-table-name</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry></descriptors><lossTolerant>false</lossTolerant><penaltyDuration>30 sec</penaltyDuration><properties><entry><key>orc-config-resources</key></entry><entry><key>orc-stripe-size</key><value>100000</value></entry><entry><key>orc-buffer-size</key><value>10000</value></entry><entry><key>orc-compression-type</key><value>NONE</value></entry><entry><key>orc-hive-table-name</key><value>orcTest</value></entry></properties><runDurationMillis>0</runDurationMillis><schedulingPeriod>0 sec</schedulingPeriod><schedulingStrategy>TIMER_DRIVEN</schedulingStrategy><yieldDuration>1 sec</yieldDuration></config><name>ConvertAvroToORC</name><relationships><autoTerminate>true</autoTerminate><description>A FlowFile is routed to this relationship if it cannot be parsed as Avro or cannot be converted to ORC for any reason</description><name>failure</name></relationships><relationships><autoTerminate>false</autoTerminate><description>A FlowFile is routed to this relationship after it has been converted to ORC format.</description><name>success</name></relationships><state>STOPPED</state><style/><supportsEventDriven>false</supportsEventDriven><supportsParallelProcessing>true</supportsParallelProcessing><type>org.apache.nifi.processors.hive.ConvertAvroToORC</type></processors><processors><id>fea02638-ee1c-4419-a9b7-5e6b1ab57380</id><parentGroupId>9007185c-8178-4606-81a5-b6a4e930ec8b</parentGroupId><position><x>574.1690141558279</x><y>16.506673020153812</y></position><config><bulletinLevel>WARN</bulletinLevel><comments></comments><concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount><defaultConcurrentTasks><entry><key>TIMER_DRIVEN</key><value>1</value></entry><entry><key>EVENT_DRIVEN</key><value>0</value></entry><entry><key>CRON_DRIVEN</key><value>1</value></entry></defaultConcurrentTasks><defaultSchedulingPeriod><entry><key>TIMER_DRIVEN</key><value>0 sec</value></entry><entry><key>CRON_DRIVEN</key><value>* * * * * ?</value></entry></defaultSchedulingPeriod><descriptors><entry><key>Regular Expression</key><value><defaultValue>(?s:^.*$)</defaultValue><description>The Search Value to search for in the FlowFile content. Only used for 'Literal Replace' and 'Regex Replace' matching strategies</description><displayName>Search Value</displayName><dynamic>false</dynamic><name>Regular Expression</name><required>true</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>Replacement Value</key><value><defaultValue>$1</defaultValue><description>The value to insert using the 'Replacement Strategy'. Using "Regex Replace" back-references to Regular Expression capturing groups are supported, but back-references that reference capturing groups that do not exist in the regular expression will be treated as literal value. Back References may also be referenced using the Expression Language, as '$1', '$2', etc. The single-tick marks MUST be included, as these variables are not "Standard" attribute names (attribute names must be quoted unless they contain only numbers, letters, and _).</description><displayName>Replacement Value</displayName><dynamic>false</dynamic><name>Replacement Value</name><required>true</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>Character Set</key><value><defaultValue>UTF-8</defaultValue><description>The Character Set in which the file is encoded</description><displayName>Character Set</displayName><dynamic>false</dynamic><name>Character Set</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Maximum Buffer Size</key><value><defaultValue>1 MB</defaultValue><description>Specifies the maximum amount of data to buffer (per file or per line, depending on the Evaluation Mode) in order to apply the replacement. If 'Entire Text' (in Evaluation Mode) is selected and the FlowFile is larger than this value, the FlowFile will be routed to 'failure'. In 'Line-by-Line' Mode, if a single line is larger than this value, the FlowFile will be routed to 'failure'. A default value of 1 MB is provided, primarily for 'Entire Text' mode. In 'Line-by-Line' Mode, a value such as 8 KB or 16 KB is suggested. This value is ignored if the <Replacement Strategy> property is set to one of: Append, Prepend, Always Replace</description><displayName>Maximum Buffer Size</displayName><dynamic>false</dynamic><name>Maximum Buffer Size</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Replacement Strategy</key><value><allowableValues><description>Insert the Replacement Value at the beginning of the FlowFile or the beginning of each line (depending on the Evaluation Mode). For "Line-by-Line" Evaluation Mode, the value will be prepended to each line. For "Entire Text" evaluation mode, the value will be prepended to the entire text.</description><displayName>Prepend</displayName><value>Prepend</value></allowableValues><allowableValues><description>Insert the Replacement Value at the end of the FlowFile or the end of each line (depending on the Evluation Mode). For "Line-by-Line" Evaluation Mode, the value will be appended to each line. For "Entire Text" evaluation mode, the value will be appended to the entire text.</description><displayName>Append</displayName><value>Append</value></allowableValues><allowableValues><description>Interpret the Search Value as a Regular Expression and replace all matches with the Replacement Value. The Replacement Value may reference Capturing Groups used in the Search Value by using a dollar-sign followed by the Capturing Group number, such as $1 or $2. If the Search Value is set to .* then everything is replaced without even evaluating the Regular Expression.</description><displayName>Regex Replace</displayName><value>Regex Replace</value></allowableValues><allowableValues><description>Search for all instances of the Search Value and replace the matches with the Replacement Value.</description><displayName>Literal Replace</displayName><value>Literal Replace</value></allowableValues><allowableValues><description>Always replaces the entire line or the entire contents of the FlowFile (depending on the value of the <Evaluation Mode> property) and does not bother searching for any value. When this strategy is chosen, the <Search Value> property is ignored.</description><displayName>Always Replace</displayName><value>Always Replace</value></allowableValues><defaultValue>Regex Replace</defaultValue><description>The strategy for how and what to replace within the FlowFile's text content.</description><displayName>Replacement Strategy</displayName><dynamic>false</dynamic><name>Replacement Strategy</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Evaluation Mode</key><value><allowableValues><displayName>Line-by-Line</displayName><value>Line-by-Line</value></allowableValues><allowableValues><displayName>Entire text</displayName><value>Entire text</value></allowableValues><defaultValue>Entire text</defaultValue><description>Run the 'Replacement Strategy' against each line separately (Line-by-Line) or buffer the entire file into memory (Entire Text) and run against that.</description><displayName>Evaluation Mode</displayName><dynamic>false</dynamic><name>Evaluation Mode</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry></descriptors><lossTolerant>false</lossTolerant><penaltyDuration>30 sec</penaltyDuration><properties><entry><key>Regular Expression</key><value>(?s:^.*$)</value></entry><entry><key>Replacement Value</key><value>${hive.ddl} LOCATION '/user/hive/orcTest'</value></entry><entry><key>Character Set</key><value>UTF-8</value></entry><entry><key>Maximum Buffer Size</key><value>1 MB</value></entry><entry><key>Replacement Strategy</key><value>Always Replace</value></entry><entry><key>Evaluation Mode</key><value>Entire text</value></entry></properties><runDurationMillis>0</runDurationMillis><schedulingPeriod>0 sec</schedulingPeriod><schedulingStrategy>TIMER_DRIVEN</schedulingStrategy><yieldDuration>1 sec</yieldDuration></config><name>ReplaceText</name><relationships><autoTerminate>true</autoTerminate><description>FlowFiles that could not be updated are routed to this relationship</description><name>failure</name></relationships><relationships><autoTerminate>false</autoTerminate><description>FlowFiles that have been successfully processed are routed to this relationship. This includes both FlowFiles that had text replaced and those that did not.</description><name>success</name></relationships><state>STOPPED</state><style/><supportsEventDriven>true</supportsEventDriven><supportsParallelProcessing>true</supportsParallelProcessing><type>org.apache.nifi.processors.standard.ReplaceText</type></processors><processors><id>17ad09e6-4c2e-4740-a046-f02433fbedc9</id><parentGroupId>9007185c-8178-4606-81a5-b6a4e930ec8b</parentGroupId><position><x>-8.739410400390625</x><y>17.682891845703125</y></position><config><bulletinLevel>WARN</bulletinLevel><comments></comments><concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount><defaultConcurrentTasks><entry><key>TIMER_DRIVEN</key><value>1</value></entry><entry><key>EVENT_DRIVEN</key><value>0</value></entry><entry><key>CRON_DRIVEN</key><value>1</value></entry></defaultConcurrentTasks><defaultSchedulingPeriod><entry><key>TIMER_DRIVEN</key><value>0 sec</value></entry><entry><key>CRON_DRIVEN</key><value>* * * * * ?</value></entry></defaultSchedulingPeriod><descriptors><entry><key>Hadoop configuration files</key><value><description>A comma-separated list of Hadoop configuration files</description><displayName>Hadoop configuration files</displayName><dynamic>false</dynamic><name>Hadoop configuration files</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Record schema</key><value><description>Outgoing Avro schema for each record created from a CSV row</description><displayName>Record schema</displayName><dynamic>false</dynamic><name>Record schema</name><required>true</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>CSV charset</key><value><defaultValue>utf8</defaultValue><description>Character set for CSV files</description><displayName>CSV charset</displayName><dynamic>false</dynamic><name>CSV charset</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>CSV delimiter</key><value><defaultValue>,</defaultValue><description>Delimiter character for CSV records</description><displayName>CSV delimiter</displayName><dynamic>false</dynamic><name>CSV delimiter</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>CSV quote character</key><value><defaultValue>"</defaultValue><description>Quote character for CSV values</description><displayName>CSV quote character</displayName><dynamic>false</dynamic><name>CSV quote character</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>CSV escape character</key><value><defaultValue>\</defaultValue><description>Escape character for CSV values</description><displayName>CSV escape character</displayName><dynamic>false</dynamic><name>CSV escape character</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Use CSV header line</key><value><defaultValue>false</defaultValue><description>Whether to use the first line as a header</description><displayName>Use CSV header line</displayName><dynamic>false</dynamic><name>Use CSV header line</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Lines to skip</key><value><defaultValue>0</defaultValue><description>Number of lines to skip before reading header or data</description><displayName>Lines to skip</displayName><dynamic>false</dynamic><name>Lines to skip</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry></descriptors><lossTolerant>false</lossTolerant><penaltyDuration>30 sec</penaltyDuration><properties><entry><key>Hadoop configuration files</key></entry><entry><key>Record schema</key><value>{
  2. "namespace": "example.avro",
  3. "type": "record",
  4. "name": "CrimeStateByState",
  5. "fields": [
  6. {"name": "State", "type": "string"},
  7. {"name": "Type_of_Crime", "type": "string"},
  8. {"name": "Crime", "type": "string"},
  9. {"name": "Year", "type": "string"},
  10. {"name": "Count", "type": "int"}
  11. ]
  12. }</value></entry><entry><key>CSV charset</key><value>utf8</value></entry><entry><key>CSV delimiter</key><value>,</value></entry><entry><key>CSV quote character</key><value>"</value></entry><entry><key>CSV escape character</key><value>\</value></entry><entry><key>Use CSV header line</key><value>false</value></entry><entry><key>Lines to skip</key><value>1</value></entry></properties><runDurationMillis>0</runDurationMillis><schedulingPeriod>0 sec</schedulingPeriod><schedulingStrategy>TIMER_DRIVEN</schedulingStrategy><yieldDuration>1 sec</yieldDuration></config><name>ConvertCSVToAvro</name><relationships><autoTerminate>true</autoTerminate><description>CSV content that could not be processed</description><name>failure</name></relationships><relationships><autoTerminate>true</autoTerminate><description>CSV content that could not be converted</description><name>incompatible</name></relationships><relationships><autoTerminate>false</autoTerminate><description>Avro content that was converted successfully from CSV</description><name>success</name></relationships><state>STOPPED</state><style/><supportsEventDriven>false</supportsEventDriven><supportsParallelProcessing>true</supportsParallelProcessing><type>org.apache.nifi.processors.kite.ConvertCSVToAvro</type></processors><processors><id>1b46938d-13fb-4e05-8334-e57f5d76edec</id><parentGroupId>9007185c-8178-4606-81a5-b6a4e930ec8b</parentGroupId><position><x>575.2002192159453</x><y>-187.76140055603284</y></position><config><bulletinLevel>WARN</bulletinLevel><comments></comments><concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount><defaultConcurrentTasks><entry><key>TIMER_DRIVEN</key><value>1</value></entry><entry><key>EVENT_DRIVEN</key><value>0</value></entry><entry><key>CRON_DRIVEN</key><value>1</value></entry></defaultConcurrentTasks><defaultSchedulingPeriod><entry><key>TIMER_DRIVEN</key><value>0 sec</value></entry><entry><key>CRON_DRIVEN</key><value>* * * * * ?</value></entry></defaultSchedulingPeriod><descriptors><entry><key>Log Level</key><value><allowableValues><displayName>trace</displayName><value>trace</value></allowableValues><allowableValues><displayName>debug</displayName><value>debug</value></allowableValues><allowableValues><displayName>info</displayName><value>info</value></allowableValues><allowableValues><displayName>warn</displayName><value>warn</value></allowableValues><allowableValues><displayName>error</displayName><value>error</value></allowableValues><defaultValue>info</defaultValue><description>The Log Level to use when logging the Attributes</description><displayName>Log Level</displayName><dynamic>false</dynamic><name>Log Level</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Log Payload</key><value><allowableValues><displayName>true</displayName><value>true</value></allowableValues><allowableValues><displayName>false</displayName><value>false</value></allowableValues><defaultValue>false</defaultValue><description>If true, the FlowFile's payload will be logged, in addition to its attributes; otherwise, just the Attributes will be logged.</description><displayName>Log Payload</displayName><dynamic>false</dynamic><name>Log Payload</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Attributes to Log</key><value><description>A comma-separated list of Attributes to Log. If not specified, all attributes will be logged.</description><displayName>Attributes to Log</displayName><dynamic>false</dynamic><name>Attributes to Log</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Attributes to Ignore</key><value><description>A comma-separated list of Attributes to ignore. If not specified, no attributes will be ignored.</description><displayName>Attributes to Ignore</displayName><dynamic>false</dynamic><name>Attributes to Ignore</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Log prefix</key><value><description>Log prefix appended to the log lines. It helps to distinguish the output of multiple LogAttribute processors.</description><displayName>Log prefix</displayName><dynamic>false</dynamic><name>Log prefix</name><required>false</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry></descriptors><lossTolerant>false</lossTolerant><penaltyDuration>30 sec</penaltyDuration><properties><entry><key>Log Level</key><value>info</value></entry><entry><key>Log Payload</key><value>true</value></entry><entry><key>Attributes to Log</key></entry><entry><key>Attributes to Ignore</key></entry><entry><key>Log prefix</key></entry></properties><runDurationMillis>0</runDurationMillis><schedulingPeriod>0 sec</schedulingPeriod><schedulingStrategy>TIMER_DRIVEN</schedulingStrategy><yieldDuration>1 sec</yieldDuration></config><name>LogAttribute</name><relationships><autoTerminate>true</autoTerminate><description>All FlowFiles are routed to this relationship</description><name>success</name></relationships><state>STOPPED</state><style/><supportsEventDriven>true</supportsEventDriven><supportsParallelProcessing>true</supportsParallelProcessing><type>org.apache.nifi.processors.standard.LogAttribute</type></processors><processors><id>e8e81b14-8c6b-4ac2-beab-31b4ed131d9a</id><parentGroupId>9007185c-8178-4606-81a5-b6a4e930ec8b</parentGroupId><position><x>571.7636783511625</x><y>244.14309171085085</y></position><config><bulletinLevel>WARN</bulletinLevel><comments></comments><concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount><defaultConcurrentTasks><entry><key>TIMER_DRIVEN</key><value>1</value></entry><entry><key>EVENT_DRIVEN</key><value>0</value></entry><entry><key>CRON_DRIVEN</key><value>1</value></entry></defaultConcurrentTasks><defaultSchedulingPeriod><entry><key>TIMER_DRIVEN</key><value>0 sec</value></entry><entry><key>CRON_DRIVEN</key><value>* * * * * ?</value></entry></defaultSchedulingPeriod><descriptors><entry><key>Hadoop Configuration Resources</key><value><description>A file or comma separated list of files which contains the Hadoop file system configuration. Without this, Hadoop will search the classpath for a 'core-site.xml' and 'hdfs-site.xml' file or will revert to a default configuration.</description><displayName>Hadoop Configuration Resources</displayName><dynamic>false</dynamic><name>Hadoop Configuration Resources</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Kerberos Principal</key><value><description>Kerberos principal to authenticate as. Requires nifi.kerberos.krb5.file to be set in your nifi.properties</description><displayName>Kerberos Principal</displayName><dynamic>false</dynamic><name>Kerberos Principal</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Kerberos Keytab</key><value><description>Kerberos keytab associated with the principal. Requires nifi.kerberos.krb5.file to be set in your nifi.properties</description><displayName>Kerberos Keytab</displayName><dynamic>false</dynamic><name>Kerberos Keytab</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Kerberos Relogin Period</key><value><defaultValue>4 hours</defaultValue><description>Period of time which should pass before attempting a kerberos relogin</description><displayName>Kerberos Relogin Period</displayName><dynamic>false</dynamic><name>Kerberos Relogin Period</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Directory</key><value><description>The parent HDFS directory to which files should be written</description><displayName>Directory</displayName><dynamic>false</dynamic><name>Directory</name><required>true</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>Conflict Resolution Strategy</key><value><allowableValues><displayName>replace</displayName><value>replace</value></allowableValues><allowableValues><displayName>ignore</displayName><value>ignore</value></allowableValues><allowableValues><displayName>fail</displayName><value>fail</value></allowableValues><defaultValue>fail</defaultValue><description>Indicates what should happen when a file with the same name already exists in the output directory</description><displayName>Conflict Resolution Strategy</displayName><dynamic>false</dynamic><name>Conflict Resolution Strategy</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Block Size</key><value><description>Size of each block as written to HDFS. This overrides the Hadoop Configuration</description><displayName>Block Size</displayName><dynamic>false</dynamic><name>Block Size</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>IO Buffer Size</key><value><description>Amount of memory to use to buffer file contents during IO. This overrides the Hadoop Configuration</description><displayName>IO Buffer Size</displayName><dynamic>false</dynamic><name>IO Buffer Size</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Replication</key><value><description>Number of times that HDFS will replicate each file. This overrides the Hadoop Configuration</description><displayName>Replication</displayName><dynamic>false</dynamic><name>Replication</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Permissions umask</key><value><description>A umask represented as an octal number which determines the permissions of files written to HDFS. This overrides the Hadoop Configuration dfs.umaskmode</description><displayName>Permissions umask</displayName><dynamic>false</dynamic><name>Permissions umask</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Remote Owner</key><value><description>Changes the owner of the HDFS file to this value after it is written. This only works if NiFi is running as a user that has HDFS super user privilege to change owner</description><displayName>Remote Owner</displayName><dynamic>false</dynamic><name>Remote Owner</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Remote Group</key><value><description>Changes the group of the HDFS file to this value after it is written. This only works if NiFi is running as a user that has HDFS super user privilege to change group</description><displayName>Remote Group</displayName><dynamic>false</dynamic><name>Remote Group</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Compression codec</key><value><allowableValues><displayName>NONE</displayName><value>NONE</value></allowableValues><allowableValues><displayName>DEFAULT</displayName><value>DEFAULT</value></allowableValues><allowableValues><displayName>BZIP</displayName><value>BZIP</value></allowableValues><allowableValues><displayName>GZIP</displayName><value>GZIP</value></allowableValues><allowableValues><displayName>LZ4</displayName><value>LZ4</value></allowableValues><allowableValues><displayName>SNAPPY</displayName><value>SNAPPY</value></allowableValues><allowableValues><displayName>AUTOMATIC</displayName><value>AUTOMATIC</value></allowableValues><defaultValue>NONE</defaultValue><description></description><displayName>Compression codec</displayName><dynamic>false</dynamic><name>Compression codec</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry></descriptors><lossTolerant>false</lossTolerant><penaltyDuration>30 sec</penaltyDuration><properties><entry><key>Hadoop Configuration Resources</key><value>/Users/mburgess/Downloads/HDFS_CLIENT-configs/core-site.xml,/Users/mburgess/Downloads/HDFS_CLIENT-configs/hdfs-site.xml</value></entry><entry><key>Kerberos Principal</key></entry><entry><key>Kerberos Keytab</key></entry><entry><key>Kerberos Relogin Period</key><value>4 hours</value></entry><entry><key>Directory</key><value>/user/hive/orcTest</value></entry><entry><key>Conflict Resolution Strategy</key><value>replace</value></entry><entry><key>Block Size</key></entry><entry><key>IO Buffer Size</key></entry><entry><key>Replication</key></entry><entry><key>Permissions umask</key><value>0666</value></entry><entry><key>Remote Owner</key><value>hive</value></entry><entry><key>Remote Group</key><value>hdfs</value></entry><entry><key>Compression codec</key><value>NONE</value></entry></properties><runDurationMillis>0</runDurationMillis><schedulingPeriod>0 sec</schedulingPeriod><schedulingStrategy>TIMER_DRIVEN</schedulingStrategy><yieldDuration>1 sec</yieldDuration></config><name>PutHDFS</name><relationships><autoTerminate>true</autoTerminate><description>Files that could not be written to HDFS for some reason are transferred to this relationship</description><name>failure</name></relationships><relationships><autoTerminate>false</autoTerminate><description>Files that have been successfully written to HDFS are transferred to this relationship</description><name>success</name></relationships><state>STOPPED</state><style/><supportsEventDriven>false</supportsEventDriven><supportsParallelProcessing>true</supportsParallelProcessing><type>org.apache.nifi.processors.hadoop.PutHDFS</type></processors><processors><id>4a2f1555-737d-4d30-a717-75f8cbdb4d61</id><parentGroupId>9007185c-8178-4606-81a5-b6a4e930ec8b</parentGroupId><position><x>-5.513196582688096</x><y>-198.47245681456002</y></position><config><bulletinLevel>WARN</bulletinLevel><comments></comments><concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount><defaultConcurrentTasks><entry><key>TIMER_DRIVEN</key><value>1</value></entry><entry><key>EVENT_DRIVEN</key><value>0</value></entry><entry><key>CRON_DRIVEN</key><value>1</value></entry></defaultConcurrentTasks><defaultSchedulingPeriod><entry><key>TIMER_DRIVEN</key><value>0 sec</value></entry><entry><key>CRON_DRIVEN</key><value>* * * * * ?</value></entry></defaultSchedulingPeriod><descriptors><entry><key>Input Directory</key><value><description>The input directory from which to pull files</description><displayName>Input Directory</displayName><dynamic>false</dynamic><name>Input Directory</name><required>true</required><sensitive>false</sensitive><supportsEl>true</supportsEl></value></entry><entry><key>File Filter</key><value><defaultValue>[^\.].*</defaultValue><description>Only files whose names match the given regular expression will be picked up</description><displayName>File Filter</displayName><dynamic>false</dynamic><name>File Filter</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Path Filter</key><value><description>When Recurse Subdirectories is true, then only subdirectories whose path matches the given regular expression will be scanned</description><displayName>Path Filter</displayName><dynamic>false</dynamic><name>Path Filter</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Batch Size</key><value><defaultValue>10</defaultValue><description>The maximum number of files to pull in each iteration</description><displayName>Batch Size</displayName><dynamic>false</dynamic><name>Batch Size</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Keep Source File</key><value><allowableValues><displayName>true</displayName><value>true</value></allowableValues><allowableValues><displayName>false</displayName><value>false</value></allowableValues><defaultValue>false</defaultValue><description>If true, the file is not deleted after it has been copied to the Content Repository; this causes the file to be picked up continually and is useful for testing purposes. If not keeping original NiFi will need write permissions on the directory it is pulling from otherwise it will ignore the file.</description><displayName>Keep Source File</displayName><dynamic>false</dynamic><name>Keep Source File</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Recurse Subdirectories</key><value><allowableValues><displayName>true</displayName><value>true</value></allowableValues><allowableValues><displayName>false</displayName><value>false</value></allowableValues><defaultValue>true</defaultValue><description>Indicates whether or not to pull files from subdirectories</description><displayName>Recurse Subdirectories</displayName><dynamic>false</dynamic><name>Recurse Subdirectories</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Polling Interval</key><value><defaultValue>0 sec</defaultValue><description>Indicates how long to wait before performing a directory listing</description><displayName>Polling Interval</displayName><dynamic>false</dynamic><name>Polling Interval</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Ignore Hidden Files</key><value><allowableValues><displayName>true</displayName><value>true</value></allowableValues><allowableValues><displayName>false</displayName><value>false</value></allowableValues><defaultValue>true</defaultValue><description>Indicates whether or not hidden files should be ignored</description><displayName>Ignore Hidden Files</displayName><dynamic>false</dynamic><name>Ignore Hidden Files</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Minimum File Age</key><value><defaultValue>0 sec</defaultValue><description>The minimum age that a file must be in order to be pulled; any file younger than this amount of time (according to last modification date) will be ignored</description><displayName>Minimum File Age</displayName><dynamic>false</dynamic><name>Minimum File Age</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Maximum File Age</key><value><description>The maximum age that a file must be in order to be pulled; any file older than this amount of time (according to last modification date) will be ignored</description><displayName>Maximum File Age</displayName><dynamic>false</dynamic><name>Maximum File Age</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Minimum File Size</key><value><defaultValue>0 B</defaultValue><description>The minimum size that a file must be in order to be pulled</description><displayName>Minimum File Size</displayName><dynamic>false</dynamic><name>Minimum File Size</name><required>true</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry><entry><key>Maximum File Size</key><value><description>The maximum size that a file can be in order to be pulled</description><displayName>Maximum File Size</displayName><dynamic>false</dynamic><name>Maximum File Size</name><required>false</required><sensitive>false</sensitive><supportsEl>false</supportsEl></value></entry></descriptors><lossTolerant>false</lossTolerant><penaltyDuration>30 sec</penaltyDuration><properties><entry><key>Input Directory</key><value>/Users/mburgess/datasets</value></entry><entry><key>File Filter</key><value>CrimeStatebyState.csv</value></entry><entry><key>Path Filter</key></entry><entry><key>Batch Size</key><value>10</value></entry><entry><key>Keep Source File</key><value>true</value></entry><entry><key>Recurse Subdirectories</key><value>true</value></entry><entry><key>Polling Interval</key><value>0 sec</value></entry><entry><key>Ignore Hidden Files</key><value>true</value></entry><entry><key>Minimum File Age</key><value>0 sec</value></entry><entry><key>Maximum File Age</key></entry><entry><key>Minimum File Size</key><value>0 B</value></entry><entry><key>Maximum File Size</key></entry></properties><runDurationMillis>0</runDurationMillis><schedulingPeriod>30 sec</schedulingPeriod><schedulingStrategy>TIMER_DRIVEN</schedulingStrategy><yieldDuration>1 sec</yieldDuration></config><name>GetFile</name><relationships><autoTerminate>false</autoTerminate><description>All files are routed to success</description><name>success</name></relationships><state>STOPPED</state><style/><supportsEventDriven>false</supportsEventDriven><supportsParallelProcessing>true</supportsParallelProcessing><type>org.apache.nifi.processors.standard.GetFile</type></processors></snippet><timestamp>05/30/2016 21:54:12 EDT</timestamp></template>
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement