API tools faq
paste
Advertisement
Guest User

Untitled

a guest
Mar 17th, 2019
129
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 14.61 KB | None | 0 0
raw download clone embed print report
  1. const { workerData, parentPort } = require('worker_threads');
  2.  
  3. const fs = require('fs');
  4. const iconv = require('iconv-lite');
  5. const util = require('util');
  6. const md5 = require('md5');
  7. const async = require('async');
  8. const parser = require('xml2js').parseString;
  9. const knex = require('knex')({
  10. client: 'pg',
  11. connection: {
  12. host : '127.0.0.1',
  13. user : 'homestead',
  14. password : 'secret',
  15. database : 'egr'
  16. }
  17. });
  18.  
  19. const MAPPING = require('../../assets/mapping/egrip_4.04');
  20. const MAX_THREADS = 200;
  21.  
  22. const DUPE_FIELDS = [
  23. 'ogrnip',
  24. 'ogrnip_data',
  25. 'egrnip',
  26. 'egrnip_data',
  27. 'egrnip_correct',
  28. 'egrnip_data_correct'
  29. ];
  30.  
  31. const PERSON_OBJECTS = [
  32. 'card',
  33. 'registration',
  34. 'location',
  35. 'email'
  36. ];
  37.  
  38. const ORGANIZATION_OBJECTS = [
  39. 'organization_register',
  40. 'info_farm_status',
  41. 'info_termination_activities',
  42. 'info_tax_organ',
  43. 'info_pension_fund',
  44. 'social_security_fund',
  45. 'info_economic_activity'
  46. ];
  47.  
  48.  
  49. asyncParseFiles(workerData, (file) => {
  50.  
  51. });
  52.  
  53.  
  54. function asyncParseFiles(filesList, cb)
  55. {
  56. let workers = 0;
  57. let filePosition = -1;
  58.  
  59.  
  60. for (let i = 0; i < filesList.length; i++) {
  61. setTimeout(fetchNextFile, 0);
  62. }
  63.  
  64.  
  65. function fetchNextFile()
  66. {
  67. if (filePosition > filesList.length - 2) return;
  68. if (workers > MAX_THREADS) return;
  69.  
  70. filePosition += 1;
  71. workers += 1;
  72.  
  73. handleFile(filesList[filePosition], (obj) => {
  74. cb(filesList[filePosition]);
  75.  
  76. workers -= 1;
  77.  
  78. setTimeout(fetchNextFile, 0);
  79. });
  80. }
  81.  
  82.  
  83. function handleFile(fileName, readcb)
  84. {
  85. fs.readFile(fileName, null, (err, content) => {
  86. parser(
  87. iconv.decode(content, 'cp1251').toString(), (err, object) => {
  88. let objects = getObjects(object);
  89.  
  90. importObjects(objects);
  91.  
  92. parentPort.postMessage(fileName);
  93. readcb();
  94. // console.log(util.inspect(objects, false, null, true));
  95. }
  96. );
  97. });
  98. }
  99. }
  100.  
  101.  
  102. async function importObjects(objects)
  103. {
  104. for (let [objectKey, object] of Object.entries(objects)) {
  105. try {
  106. let fileID = await importObject(object['data'], objectKey, MAPPING[objectKey]['tableName']);
  107.  
  108. for (let documentIndex in object['child']['documents']) {
  109. let document = object['child']['documents'][documentIndex];
  110.  
  111. if (document.hasOwnProperty('person')) {
  112. let personID = await importObject(document['person']['data'], 'person', MAPPING['person']['tableName'], 'file_id', fileID);
  113.  
  114. for (let [pOKey, personObject] of PERSON_OBJECTS.entries()) {
  115. await importObject(document['person']['child'][personObject], personObject, MAPPING[personObject]['tableName'], 'person_id', personID);
  116. }
  117.  
  118. let organizationID = await importObject(document['organization']['data'], 'organization', MAPPING['organization']['tableName'], 'person_id', personID);
  119.  
  120. for (let [oKey, organizationObject] of ORGANIZATION_OBJECTS.entries()) {
  121. await importObject(document['organization']['child'][organizationObject], organizationObject, MAPPING[organizationObject]['tableName'], 'organization_id', organizationID);
  122. }
  123.  
  124. if (document['organization']['child']['licenses'].length > 0) {
  125. let lisencesObjects = document['organization']['child']['licenses'];
  126.  
  127. for (let [licenseIndex, lisenceObject] of Object.entries(lisencesObjects)) {
  128. await importObject(lisenceObject, 'licenses', MAPPING['licenses']['tableName'], 'organization_id', organizationID);
  129. }
  130. }
  131.  
  132. if (document['organization']['child']['egrip_notes'].length > 0) {
  133. let noteObjects = document['organization']['child']['egrip_notes'];
  134.  
  135. for (let [noteIndex, noteObject] of Object.entries(noteObjects)) {
  136. let noteID = await importObject(noteObject['data'], 'egrip_notes', MAPPING['egrip_notes']['tableName'], 'organization_id', organizationID);
  137.  
  138. if (noteObject['child'].length > 0) {
  139. let noteDocumentObjects = noteObject['child'];
  140.  
  141. for (let [noteDocumentIndex, noteDocumentObject] of Object.entries(noteDocumentObjects)) {
  142. await importObject(noteDocumentObject, 'egrip_notes_documents', MAPPING['egrip_notes_documents']['tableName'], 'organization_note', noteID);
  143. }
  144. }
  145. }
  146. }
  147. }
  148. }
  149. } catch (e) {
  150.  
  151. }
  152. }
  153. // for (let [objectKey, object] of Object.entries(objects)) {
  154. // console.log('Обработка 1');
  155. // try {
  156. // importObject(object['data'], objectKey, MAPPING[objectKey]['tableName'], (fileID) => {
  157. // for (let documentIndex in object['child']['documents']) {
  158. // setTimeout(() => {
  159. // let document = object['child']['documents'][documentIndex];
  160. //
  161. // if (document.hasOwnProperty('person')) {
  162. // importObject(document['person']['data'], 'person', MAPPING['person']['tableName'], (personID) => {
  163. // for (let [pOKey, personObject] of PERSON_OBJECTS.entries()) {
  164. // setTimeout(() => {
  165. // console.log('Обработка 2');
  166. // importObject(document['person']['child'][personObject], personObject, MAPPING[personObject]['tableName'], () => {
  167. // }, 'person_id', personID);
  168. // }, 5000);
  169. // }
  170. //
  171. // // importObject(document['organization']['data'], 'organization', MAPPING['organization']['tableName'], (organizationID) => {
  172. // // for (let [oKey, organizationObject] of ORGANIZATION_OBJECTS.entries()) {
  173. // // importObject(document['organization']['child'][organizationObject], organizationObject, MAPPING[organizationObject]['tableName'], () => {
  174. // // }, 'organization_id', organizationID);
  175. // // }
  176. // //
  177. // // if (document['organization']['child']['licenses'].length > 0) {
  178. // // let lisencesObjects = document['organization']['child']['licenses'];
  179. // //
  180. // // for (let [licenseIndex, lisenceObject] of Object.entries(lisencesObjects)) {
  181. // // importObject(lisenceObject, 'licenses', MAPPING['licenses']['tableName'], () => {
  182. // // }, 'organization_id', organizationID);
  183. // // }
  184. // // }
  185. // //
  186. // // if (document['organization']['child']['egrip_notes'].length > 0) {
  187. // // let noteObjects = document['organization']['child']['egrip_notes'];
  188. // //
  189. // // for (let [noteIndex, noteObject] of Object.entries(noteObjects)) {
  190. // // importObject(noteObject['data'], 'egrip_notes', MAPPING['egrip_notes']['tableName'], (noteID) => {
  191. // // if (noteObject['child'].length > 0) {
  192. // // let noteDocumentObjects = noteObject['child'];
  193. // //
  194. // // for (let [noteDocumentIndex, noteDocumentObject] of Object.entries(noteDocumentObjects)) {
  195. // // importObject(noteDocumentObject, 'egrip_notes_documents', MAPPING['egrip_notes_documents']['tableName'], () => {
  196. // // }, 'organization_note', noteID);
  197. // // }
  198. // // }
  199. // // }, 'organization_id', organizationID);
  200. // // }
  201. // // }
  202. // // }, 'person_id', personID);
  203. // }, 'file_id', fileID);
  204. // }
  205. // }, 2000);
  206. // }
  207. // });
  208. // } catch (e) {
  209. // ;
  210. // }
  211. // }
  212. }
  213.  
  214.  
  215. async function importObject(object, objectKey, table, foreignColumn = null, foreignID = null)
  216. {
  217. let check = object;
  218.  
  219. for (let dupeField in DUPE_FIELDS) {
  220. if (check.hasOwnProperty(dupeField)) {
  221. delete check[dupeField];
  222. }
  223. }
  224.  
  225. if (MAPPING[objectKey].hasOwnProperty('foreign')) {
  226. let foreignColumn = MAPPING[objectKey]['foreign']['column'];
  227.  
  228. if (check.hasOwnProperty(foreignColumn)) {
  229. delete check[foreignColumn];
  230. }
  231. }
  232.  
  233. let hash = md5(
  234. JSON.stringify(
  235. Object.assign({}, check)
  236. ).toString()
  237. );
  238.  
  239. let checkData = await knex(table).where({hash: hash}).select('id');
  240.  
  241. if (checkData.length === 0) {
  242. if (foreignColumn != null && foreignID != null) {
  243. object[foreignColumn] = foreignID;
  244. }
  245.  
  246. object['hash'] = hash;
  247.  
  248. let row = await knex(table).insert(Object.assign({}, object), 'id');
  249.  
  250. return row;
  251. } else {
  252. return checkData[0].id;
  253. }
  254. }
  255.  
  256.  
  257. function getObjects(object)
  258. {
  259. let objects = [];
  260.  
  261. objects['file'] = [];
  262. objects['file']['child'] = [];
  263. objects['file']['child']['documents'] = [];
  264. objects['file']['data'] = getObject(object, MAPPING['file']);
  265.  
  266. let documentObjects = new Map(Object.entries(object['Файл']['Документ']));
  267.  
  268. for (const [dKey, _documentObject] of documentObjects.entries()) {
  269. let documentObject = [];
  270. documentObject['person'] = [];
  271. documentObject['person']['child'] = [];
  272. documentObject['person']['data'] = getObject(_documentObject, MAPPING['person']);
  273.  
  274. for (let [pKey, personObject] of PERSON_OBJECTS.entries()) {
  275. documentObject['person']['child'][personObject] = getObject(_documentObject, MAPPING[personObject]);
  276. }
  277.  
  278.  
  279. documentObject['organization'] = [];
  280. documentObject['organization']['child'] = [];
  281. documentObject['organization']['child']['licenses'] = [];
  282. documentObject['organization']['child']['egrip_notes'] = [];
  283. documentObject['organization']['data'] = getObject(_documentObject, MAPPING['organization']);
  284.  
  285. for (let [oKey, organizationObject] of ORGANIZATION_OBJECTS.entries()) {
  286. documentObject['organization']['child'][organizationObject] = getObject(_documentObject, MAPPING[organizationObject]);
  287. }
  288.  
  289.  
  290. if (_documentObject['СвИП'][0].hasOwnProperty('СвЛицензия')) {
  291. let licenseObjects = new Map(Object.entries(_documentObject['СвИП'][0]['СвЛицензия']));
  292.  
  293. for (const [lKey, _licenseObject] of licenseObjects.entries()) {
  294. documentObject['organization']['child']['licenses'].push(
  295. getObject(_licenseObject, MAPPING['licenses'])
  296. );
  297. }
  298. }
  299.  
  300.  
  301. if (_documentObject['СвИП'][0].hasOwnProperty('СвЗапЕГРИП')) {
  302. let noteObjects = new Map(Object.entries(_documentObject['СвИП'][0]['СвЗапЕГРИП']));
  303.  
  304. for (const [nKey, _noteObject] of noteObjects.entries()) {
  305. let noteObject = [];
  306. noteObject['child'] = [];
  307. noteObject['data'] = getObject(_noteObject, MAPPING['egrip_notes']);
  308.  
  309.  
  310. if (_noteObject.hasOwnProperty('СведПредДок')) {
  311. let noteDocumentObjects = new Map(Object.entries(_noteObject['СведПредДок']));
  312.  
  313. for (const [nDKey, _noteDocumentObject] of noteDocumentObjects.entries()) {
  314. noteObject['child'].push(
  315. getObject(_noteDocumentObject, MAPPING['egrip_notes_documents'])
  316. );
  317. }
  318. }
  319.  
  320.  
  321. documentObject['organization']['child']['egrip_notes'].push(noteObject);
  322. }
  323. }
  324.  
  325.  
  326. objects['file']['child']['documents'].push(documentObject);
  327. }
  328.  
  329. return objects;
  330. }
  331.  
  332.  
  333. function getObject(object, mapping)
  334. {
  335. let list = [];
  336.  
  337. for (let field in mapping['fields']) {
  338. let path = getPath(mapping['fields'][field]['xpath'].split('/'));
  339.  
  340. try {
  341. let value = eval('object' + path);
  342.  
  343. if (value !== undefined) {
  344. list[field] = value;
  345. }
  346. } catch (e) {
  347. continue;
  348. }
  349. }
  350.  
  351. return list;
  352. }
  353.  
  354.  
  355. function getPath(list)
  356. {
  357. let result = '';
  358.  
  359. for (let item in list) {
  360. let key = list[item];
  361.  
  362. if (key.charAt(0) === '@') {
  363. result += `['$']['${key.replace('@', '')}']`;
  364. } else {
  365. result += key === '0' ? '[0]' : `['${key}']`;
  366. }
  367. }
  368.  
  369. return result;
  370. }
  371.  
  372.  
  373. function getTime(timeStart = null)
  374. {
  375. let time = Math.floor(new Date().getTime() / 1000.0);
  376.  
  377. return timeStart === null
  378. ? time
  379. : time - timeStart;
  380. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!