API tools faq
paste
Advertisement
Guest User

Untitled

a guest
Oct 10th, 2016
116
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 18.73 KB | None | 0 0
raw download clone embed print report
  1. <?xml version="1.0" ?>
  2. <template encoding-version="1.0">
  3. <description>This template uses ExecuteScript and Groovy to read in (from a file) a list of tables to fetch, then if the incoming table name (from ListDatabaseTables) is in the list of tables to fetch, sends the flow file to success (for use by ExecuteSQL) or failure</description>
  4. <groupId>af725e75-0157-1000-3844-d085884a56db</groupId>
  5. <name>LookupFilter</name>
  6. <snippet>
  7. <connections>
  8. <id>af73282a-0157-1000-0000-000000000000</id>
  9. <parentGroupId>af725e75-0157-1000-0000-000000000000</parentGroupId>
  10. <backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
  11. <backPressureObjectThreshold>10000</backPressureObjectThreshold>
  12. <destination>
  13. <groupId>af725e75-0157-1000-0000-000000000000</groupId>
  14. <id>af727e79-0157-1000-0000-000000000000</id>
  15. <type>PROCESSOR</type>
  16. </destination>
  17. <flowFileExpiration>0 sec</flowFileExpiration>
  18. <labelIndex>1</labelIndex>
  19. <name></name>
  20. <selectedRelationships>success</selectedRelationships>
  21. <source>
  22. <groupId>af725e75-0157-1000-0000-000000000000</groupId>
  23. <id>af731ca8-0157-1000-0000-000000000000</id>
  24. <type>PROCESSOR</type>
  25. </source>
  26. <zIndex>0</zIndex>
  27. </connections>
  28. <connections>
  29. <id>af74752e-0157-1000-0000-000000000000</id>
  30. <parentGroupId>af725e75-0157-1000-0000-000000000000</parentGroupId>
  31. <backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
  32. <backPressureObjectThreshold>10000</backPressureObjectThreshold>
  33. <destination>
  34. <groupId>af725e75-0157-1000-0000-000000000000</groupId>
  35. <id>af7463eb-0157-1000-0000-000000000000</id>
  36. <type>PROCESSOR</type>
  37. </destination>
  38. <flowFileExpiration>0 sec</flowFileExpiration>
  39. <labelIndex>1</labelIndex>
  40. <name></name>
  41. <selectedRelationships>failure</selectedRelationships>
  42. <source>
  43. <groupId>af725e75-0157-1000-0000-000000000000</groupId>
  44. <id>af727e79-0157-1000-0000-000000000000</id>
  45. <type>PROCESSOR</type>
  46. </source>
  47. <zIndex>0</zIndex>
  48. </connections>
  49. <connections>
  50. <id>af7b4775-0157-1000-0000-000000000000</id>
  51. <parentGroupId>af725e75-0157-1000-0000-000000000000</parentGroupId>
  52. <backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
  53. <backPressureObjectThreshold>10000</backPressureObjectThreshold>
  54. <destination>
  55. <groupId>af725e75-0157-1000-0000-000000000000</groupId>
  56. <id>af7b00b9-0157-1000-0000-000000000000</id>
  57. <type>PROCESSOR</type>
  58. </destination>
  59. <flowFileExpiration>0 sec</flowFileExpiration>
  60. <labelIndex>1</labelIndex>
  61. <name></name>
  62. <selectedRelationships>success</selectedRelationships>
  63. <source>
  64. <groupId>af725e75-0157-1000-0000-000000000000</groupId>
  65. <id>af727e79-0157-1000-0000-000000000000</id>
  66. <type>PROCESSOR</type>
  67. </source>
  68. <zIndex>0</zIndex>
  69. </connections>
  70. <connections>
  71. <id>af7b5d29-0157-1000-0000-000000000000</id>
  72. <parentGroupId>af725e75-0157-1000-0000-000000000000</parentGroupId>
  73. <backPressureDataSizeThreshold>1 GB</backPressureDataSizeThreshold>
  74. <backPressureObjectThreshold>10000</backPressureObjectThreshold>
  75. <destination>
  76. <groupId>af725e75-0157-1000-0000-000000000000</groupId>
  77. <id>af729a11-0157-1000-0000-000000000000</id>
  78. <type>PROCESSOR</type>
  79. </destination>
  80. <flowFileExpiration>0 sec</flowFileExpiration>
  81. <labelIndex>1</labelIndex>
  82. <name></name>
  83. <selectedRelationships>success</selectedRelationships>
  84. <source>
  85. <groupId>af725e75-0157-1000-0000-000000000000</groupId>
  86. <id>af7b00b9-0157-1000-0000-000000000000</id>
  87. <type>PROCESSOR</type>
  88. </source>
  89. <zIndex>0</zIndex>
  90. </connections>
  91. <controllerServices>
  92. <id>b28b426b-0156-1000-0000-000000000000</id>
  93. <parentGroupId>af725e75-0157-1000-0000-000000000000</parentGroupId>
  94. <comments></comments>
  95. <descriptors>
  96. <entry>
  97. <key>Database Connection URL</key>
  98. <value>
  99. <name>Database Connection URL</name>
  100. </value>
  101. </entry>
  102. <entry>
  103. <key>Database Driver Class Name</key>
  104. <value>
  105. <name>Database Driver Class Name</name>
  106. </value>
  107. </entry>
  108. <entry>
  109. <key>database-driver-locations</key>
  110. <value>
  111. <name>database-driver-locations</name>
  112. </value>
  113. </entry>
  114. <entry>
  115. <key>Database User</key>
  116. <value>
  117. <name>Database User</name>
  118. </value>
  119. </entry>
  120. <entry>
  121. <key>Password</key>
  122. <value>
  123. <name>Password</name>
  124. </value>
  125. </entry>
  126. <entry>
  127. <key>Max Wait Time</key>
  128. <value>
  129. <name>Max Wait Time</name>
  130. </value>
  131. </entry>
  132. <entry>
  133. <key>Max Total Connections</key>
  134. <value>
  135. <name>Max Total Connections</name>
  136. </value>
  137. </entry>
  138. <entry>
  139. <key>Validation-query</key>
  140. <value>
  141. <name>Validation-query</name>
  142. </value>
  143. </entry>
  144. </descriptors>
  145. <name>MySQLConnectionPool</name>
  146. <properties>
  147. <entry>
  148. <key>Database Connection URL</key>
  149. <value>jdbc:mysql://192.168.99.100:32768/test_schema</value>
  150. </entry>
  151. <entry>
  152. <key>Database Driver Class Name</key>
  153. <value>com.mysql.jdbc.Driver</value>
  154. </entry>
  155. <entry>
  156. <key>database-driver-locations</key>
  157. <value>file:///Users/mburgess/jdbc_drivers/mysql-connector-java-5.1.38-bin.jar</value>
  158. </entry>
  159. <entry>
  160. <key>Database User</key>
  161. <value>root</value>
  162. </entry>
  163. <entry>
  164. <key>Password</key>
  165. </entry>
  166. <entry>
  167. <key>Max Wait Time</key>
  168. <value>500 millis</value>
  169. </entry>
  170. <entry>
  171. <key>Max Total Connections</key>
  172. <value>8</value>
  173. </entry>
  174. <entry>
  175. <key>Validation-query</key>
  176. </entry>
  177. </properties>
  178. <state>ENABLED</state>
  179. <type>org.apache.nifi.dbcp.DBCPConnectionPool</type>
  180. </controllerServices>
  181. <processors>
  182. <id>af727e79-0157-1000-0000-000000000000</id>
  183. <parentGroupId>af725e75-0157-1000-0000-000000000000</parentGroupId>
  184. <position>
  185. <x>574.0</x>
  186. <y>0.0</y>
  187. </position>
  188. <config>
  189. <bulletinLevel>WARN</bulletinLevel>
  190. <comments></comments>
  191. <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
  192. <descriptors>
  193. <entry>
  194. <key>Script Engine</key>
  195. <value>
  196. <name>Script Engine</name>
  197. </value>
  198. </entry>
  199. <entry>
  200. <key>Script File</key>
  201. <value>
  202. <name>Script File</name>
  203. </value>
  204. </entry>
  205. <entry>
  206. <key>Script Body</key>
  207. <value>
  208. <name>Script Body</name>
  209. </value>
  210. </entry>
  211. <entry>
  212. <key>Module Directory</key>
  213. <value>
  214. <name>Module Directory</name>
  215. </value>
  216. </entry>
  217. </descriptors>
  218. <lossTolerant>false</lossTolerant>
  219. <penaltyDuration>30 sec</penaltyDuration>
  220. <properties>
  221. <entry>
  222. <key>Script Engine</key>
  223. <value>Groovy</value>
  224. </entry>
  225. <entry>
  226. <key>Script File</key>
  227. </entry>
  228. <entry>
  229. <key>Script Body</key>
  230. <value>def flowFile = session.get()
  231. if(!flowFile) return
  232.  
  233. // Read filter into a set
  234. def tableFilter = [] as Set<String>
  235. new File('/Users/mburgess/filter.txt').eachLine { line ->
  236. tableFilter << line
  237. }
  238.  
  239. session.transfer(flowFile, tableFilter.contains(flowFile.getAttribute('db.table.name')) ? REL_SUCCESS : REL_FAILURE)</value>
  240. </entry>
  241. <entry>
  242. <key>Module Directory</key>
  243. </entry>
  244. </properties>
  245. <runDurationMillis>0</runDurationMillis>
  246. <schedulingPeriod>0 sec</schedulingPeriod>
  247. <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
  248. <yieldDuration>1 sec</yieldDuration>
  249. </config>
  250. <name>ExecuteScript</name>
  251. <relationships>
  252. <autoTerminate>false</autoTerminate>
  253. <name>failure</name>
  254. </relationships>
  255. <relationships>
  256. <autoTerminate>false</autoTerminate>
  257. <name>success</name>
  258. </relationships>
  259. <style></style>
  260. <type>org.apache.nifi.processors.script.ExecuteScript</type>
  261. </processors>
  262. <processors>
  263. <id>af729a11-0157-1000-0000-000000000000</id>
  264. <parentGroupId>af725e75-0157-1000-0000-000000000000</parentGroupId>
  265. <position>
  266. <x>576.0</x>
  267. <y>466.0</y>
  268. </position>
  269. <config>
  270. <bulletinLevel>WARN</bulletinLevel>
  271. <comments></comments>
  272. <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
  273. <descriptors>
  274. <entry>
  275. <key>Log Level</key>
  276. <value>
  277. <name>Log Level</name>
  278. </value>
  279. </entry>
  280. <entry>
  281. <key>Log Payload</key>
  282. <value>
  283. <name>Log Payload</name>
  284. </value>
  285. </entry>
  286. <entry>
  287. <key>Attributes to Log</key>
  288. <value>
  289. <name>Attributes to Log</name>
  290. </value>
  291. </entry>
  292. <entry>
  293. <key>Attributes to Ignore</key>
  294. <value>
  295. <name>Attributes to Ignore</name>
  296. </value>
  297. </entry>
  298. <entry>
  299. <key>Log prefix</key>
  300. <value>
  301. <name>Log prefix</name>
  302. </value>
  303. </entry>
  304. </descriptors>
  305. <lossTolerant>false</lossTolerant>
  306. <penaltyDuration>30 sec</penaltyDuration>
  307. <properties>
  308. <entry>
  309. <key>Log Level</key>
  310. <value>info</value>
  311. </entry>
  312. <entry>
  313. <key>Log Payload</key>
  314. <value>false</value>
  315. </entry>
  316. <entry>
  317. <key>Attributes to Log</key>
  318. </entry>
  319. <entry>
  320. <key>Attributes to Ignore</key>
  321. </entry>
  322. <entry>
  323. <key>Log prefix</key>
  324. </entry>
  325. </properties>
  326. <runDurationMillis>0</runDurationMillis>
  327. <schedulingPeriod>0 sec</schedulingPeriod>
  328. <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
  329. <yieldDuration>1 sec</yieldDuration>
  330. </config>
  331. <name>Do other stuff</name>
  332. <relationships>
  333. <autoTerminate>true</autoTerminate>
  334. <name>success</name>
  335. </relationships>
  336. <style></style>
  337. <type>org.apache.nifi.processors.standard.LogAttribute</type>
  338. </processors>
  339. <processors>
  340. <id>af731ca8-0157-1000-0000-000000000000</id>
  341. <parentGroupId>af725e75-0157-1000-0000-000000000000</parentGroupId>
  342. <position>
  343. <x>0.0</x>
  344. <y>1.0</y>
  345. </position>
  346. <config>
  347. <bulletinLevel>WARN</bulletinLevel>
  348. <comments></comments>
  349. <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
  350. <descriptors>
  351. <entry>
  352. <key>list-db-tables-db-connection</key>
  353. <value>
  354. <identifiesControllerService>org.apache.nifi.dbcp.DBCPService</identifiesControllerService>
  355. <name>list-db-tables-db-connection</name>
  356. </value>
  357. </entry>
  358. <entry>
  359. <key>list-db-tables-catalog</key>
  360. <value>
  361. <name>list-db-tables-catalog</name>
  362. </value>
  363. </entry>
  364. <entry>
  365. <key>list-db-tables-schema-pattern</key>
  366. <value>
  367. <name>list-db-tables-schema-pattern</name>
  368. </value>
  369. </entry>
  370. <entry>
  371. <key>list-db-tables-name-pattern</key>
  372. <value>
  373. <name>list-db-tables-name-pattern</name>
  374. </value>
  375. </entry>
  376. <entry>
  377. <key>list-db-tables-types</key>
  378. <value>
  379. <name>list-db-tables-types</name>
  380. </value>
  381. </entry>
  382. <entry>
  383. <key>list-db-include-count</key>
  384. <value>
  385. <name>list-db-include-count</name>
  386. </value>
  387. </entry>
  388. <entry>
  389. <key>list-db-refresh-interval</key>
  390. <value>
  391. <name>list-db-refresh-interval</name>
  392. </value>
  393. </entry>
  394. </descriptors>
  395. <lossTolerant>false</lossTolerant>
  396. <penaltyDuration>30 sec</penaltyDuration>
  397. <properties>
  398. <entry>
  399. <key>list-db-tables-db-connection</key>
  400. <value>b28b426b-0156-1000-0000-000000000000</value>
  401. </entry>
  402. <entry>
  403. <key>list-db-tables-catalog</key>
  404. </entry>
  405. <entry>
  406. <key>list-db-tables-schema-pattern</key>
  407. </entry>
  408. <entry>
  409. <key>list-db-tables-name-pattern</key>
  410. </entry>
  411. <entry>
  412. <key>list-db-tables-types</key>
  413. <value>TABLE</value>
  414. </entry>
  415. <entry>
  416. <key>list-db-include-count</key>
  417. <value>false</value>
  418. </entry>
  419. <entry>
  420. <key>list-db-refresh-interval</key>
  421. <value>0 sec</value>
  422. </entry>
  423. </properties>
  424. <runDurationMillis>0</runDurationMillis>
  425. <schedulingPeriod>0 sec</schedulingPeriod>
  426. <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
  427. <yieldDuration>1 sec</yieldDuration>
  428. </config>
  429. <name>ListDatabaseTables</name>
  430. <relationships>
  431. <autoTerminate>false</autoTerminate>
  432. <name>success</name>
  433. </relationships>
  434. <style></style>
  435. <type>org.apache.nifi.processors.standard.ListDatabaseTables</type>
  436. </processors>
  437. <processors>
  438. <id>af7463eb-0157-1000-0000-000000000000</id>
  439. <parentGroupId>af725e75-0157-1000-0000-000000000000</parentGroupId>
  440. <position>
  441. <x>23.0</x>
  442. <y>251.0</y>
  443. </position>
  444. <config>
  445. <bulletinLevel>WARN</bulletinLevel>
  446. <comments></comments>
  447. <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
  448. <descriptors>
  449. <entry>
  450. <key>Log Level</key>
  451. <value>
  452. <name>Log Level</name>
  453. </value>
  454. </entry>
  455. <entry>
  456. <key>Log Payload</key>
  457. <value>
  458. <name>Log Payload</name>
  459. </value>
  460. </entry>
  461. <entry>
  462. <key>Attributes to Log</key>
  463. <value>
  464. <name>Attributes to Log</name>
  465. </value>
  466. </entry>
  467. <entry>
  468. <key>Attributes to Ignore</key>
  469. <value>
  470. <name>Attributes to Ignore</name>
  471. </value>
  472. </entry>
  473. <entry>
  474. <key>Log prefix</key>
  475. <value>
  476. <name>Log prefix</name>
  477. </value>
  478. </entry>
  479. </descriptors>
  480. <lossTolerant>false</lossTolerant>
  481. <penaltyDuration>30 sec</penaltyDuration>
  482. <properties>
  483. <entry>
  484. <key>Log Level</key>
  485. <value>info</value>
  486. </entry>
  487. <entry>
  488. <key>Log Payload</key>
  489. <value>false</value>
  490. </entry>
  491. <entry>
  492. <key>Attributes to Log</key>
  493. </entry>
  494. <entry>
  495. <key>Attributes to Ignore</key>
  496. </entry>
  497. <entry>
  498. <key>Log prefix</key>
  499. </entry>
  500. </properties>
  501. <runDurationMillis>0</runDurationMillis>
  502. <schedulingPeriod>0 sec</schedulingPeriod>
  503. <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
  504. <yieldDuration>1 sec</yieldDuration>
  505. </config>
  506. <name>Blacklisted tables</name>
  507. <relationships>
  508. <autoTerminate>true</autoTerminate>
  509. <name>success</name>
  510. </relationships>
  511. <style></style>
  512. <type>org.apache.nifi.processors.standard.LogAttribute</type>
  513. </processors>
  514. <processors>
  515. <id>af7b00b9-0157-1000-0000-000000000000</id>
  516. <parentGroupId>af725e75-0157-1000-0000-000000000000</parentGroupId>
  517. <position>
  518. <x>574.0</x>
  519. <y>254.0</y>
  520. </position>
  521. <config>
  522. <bulletinLevel>WARN</bulletinLevel>
  523. <comments></comments>
  524. <concurrentlySchedulableTaskCount>1</concurrentlySchedulableTaskCount>
  525. <descriptors>
  526. <entry>
  527. <key>Database Connection Pooling Service</key>
  528. <value>
  529. <identifiesControllerService>org.apache.nifi.dbcp.DBCPService</identifiesControllerService>
  530. <name>Database Connection Pooling Service</name>
  531. </value>
  532. </entry>
  533. <entry>
  534. <key>SQL select query</key>
  535. <value>
  536. <name>SQL select query</name>
  537. </value>
  538. </entry>
  539. <entry>
  540. <key>Max Wait Time</key>
  541. <value>
  542. <name>Max Wait Time</name>
  543. </value>
  544. </entry>
  545. <entry>
  546. <key>dbf-normalize</key>
  547. <value>
  548. <name>dbf-normalize</name>
  549. </value>
  550. </entry>
  551. </descriptors>
  552. <lossTolerant>false</lossTolerant>
  553. <penaltyDuration>30 sec</penaltyDuration>
  554. <properties>
  555. <entry>
  556. <key>Database Connection Pooling Service</key>
  557. <value>b28b426b-0156-1000-0000-000000000000</value>
  558. </entry>
  559. <entry>
  560. <key>SQL select query</key>
  561. <value>select * from ${db.table.name}</value>
  562. </entry>
  563. <entry>
  564. <key>Max Wait Time</key>
  565. <value>0 seconds</value>
  566. </entry>
  567. <entry>
  568. <key>dbf-normalize</key>
  569. <value>false</value>
  570. </entry>
  571. </properties>
  572. <runDurationMillis>0</runDurationMillis>
  573. <schedulingPeriod>0 sec</schedulingPeriod>
  574. <schedulingStrategy>TIMER_DRIVEN</schedulingStrategy>
  575. <yieldDuration>1 sec</yieldDuration>
  576. </config>
  577. <name>Whitelisted tables</name>
  578. <relationships>
  579. <autoTerminate>true</autoTerminate>
  580. <name>failure</name>
  581. </relationships>
  582. <relationships>
  583. <autoTerminate>false</autoTerminate>
  584. <name>success</name>
  585. </relationships>
  586. <style></style>
  587. <type>org.apache.nifi.processors.standard.ExecuteSQL</type>
  588. </processors>
  589. </snippet>
  590. <timestamp>10/10/2016 12:49:12 EDT</timestamp>
  591. </template>
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!