Guest User

Untitled

a guest
Dec 12th, 2018
125
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. import scrap = require('./scrapping_data');
  2.  
  3. import "isomorphic-fetch"
  4. import {DateTimeFormatter, LocalDate, LocalDateTime} from 'js-joda';
  5. import fs = require('fs');
  6. const slug = require('slug');
  7.  
  8. const API_URL = 'https://dartngo-api-production.herokuapp.com';
  9. // const API_TOKEN = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6eyJfaWQiOiJ1c3JfMjVkNTI1MDYtZTlmYy00NDVhLTkxNjQtMGZhN2FlNGMwZWYyIn0sImlhdCI6MTU0NDYwNzcyNCwiZXhwIjoxNTQ0ODY2OTI0fQ._CckpVxCNED2AdUJ6ECBdTXOo3nw-8nCu6Xkq9jUCB0';
  10.  
  11. interface IUser extends Object {
  12.     email: string;
  13.     password: string;
  14.     firstName: string;
  15.     lastName: string;
  16.     dateOfBirth: string;
  17. }
  18.  
  19. interface ILandmark extends Object {
  20.     address: {
  21.         streetNumber?: string,
  22.         route?: string,
  23.         locality: string,
  24.         administrativeAreaLevel1?: string,
  25.         administrativeAreaLevel2?: string,
  26.         country: string,
  27.         postalCode?: string
  28.         longitude: number,
  29.         latitude: number
  30.     };
  31.     name: {
  32.         fr: string,
  33.     };
  34. }
  35.  
  36. interface IExperience extends Object {
  37.     id?: string;
  38.     experienceId?: string;
  39.     experienceVersionId?: string;
  40.     hostId?: string;
  41.     landmarkId?: string;
  42.     title: {
  43.         fr: string
  44.     },
  45.     slug: {
  46.         fr: string
  47.     },
  48.     publicationStatus?: string;
  49.     draftDate?: string;
  50.     experienceFormat: {
  51.         formatLabel: {
  52.             fr: string,
  53.         },
  54.         sessionFormat?: string
  55.     };
  56.     timeZoneId?: string,
  57.     defaultLanguage?: string;
  58.     expiryDate?: string;
  59.     supersededExperienceVersionId?: string;
  60.     supersedingExperienceVersionId?: string;
  61.     description: {
  62.         fr: string
  63.     };
  64.     openingTimes: Object;
  65.     pictureUrls?: [string];
  66.     weeklySessionPattern?: any;
  67.     sessionParameters: {
  68.         timeInMinutes: Number,
  69.         maximumNumberOfParticipants: Number,
  70.         durationType?: string
  71.     };
  72.     pricing: {
  73.         prices: Map<string, Object>,
  74.         extras?: [{
  75.             extraRef: string,
  76.             label: [
  77.                 {
  78.                     language: string,
  79.                     value: string
  80.                 }
  81.                 ],
  82.             amountInCents: Number
  83.         }]
  84.     };
  85.     appropriateVisitorClasses?: [string];
  86.     travelInformation?: [
  87.         {
  88.             language: string,
  89.             value: string
  90.         }
  91.         ];
  92.     commencementDate: string;
  93.     unavailabilityReason?: string;
  94. }
  95.  
  96. interface Iids extends Object {
  97.     hostId: string,
  98.     landmarkId: string,
  99.     imagesIds: [string]
  100. }
  101.  
  102. // const sleep = async (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
  103.  
  104. const isoToLocalTime = (isoDate: string) => {
  105.     const d = LocalDateTime.parse(isoDate);
  106.     return d.format((DateTimeFormatter.ofPattern('HH:mm')));
  107. };
  108.  
  109. const makePostRequest = async (url: string, data: any) => {
  110.     return fetch(url, {
  111.         method: 'POST', // or 'PUT'
  112.         body: JSON.stringify(data), // data can be `string` or {object}!
  113.         headers: {
  114.             'Content-Type': 'application/json',
  115.             'Authorization': 'Token eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6eyJfaWQiOiJ1c3JfMjVkNTI1MDYtZTlmYy00NDVhLTkxNjQtMGZhN2FlNGMwZWYyIn0sImlhdCI6MTU0NDYwNzcyNCwiZXhwIjoxNTQ0ODY2OTI0fQ._CckpVxCNED2AdUJ6ECBdTXOo3nw-8nCu6Xkq9jUCB0'
  116.         }
  117.     })
  118.         .then(res => res.json())
  119.         .catch(error => error);
  120. };
  121.  
  122. const generateSchedule = (schedule: any) => {
  123.     const data = schedule[0].periods[0].defaults;
  124.     const weekDays: string[] = ["monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday"];
  125.     const weeklySchedules: { [k: string]: any } = {};
  126.     const findDay = (day: string) => {
  127.         const daySchedule = data.find((elem: any) => elem.dayOfTheWeek === day.toUpperCase() && !!elem.times && !!elem.times[0]);
  128.         return !!daySchedule ? [{
  129.             startTime: isoToLocalTime(daySchedule.times[0].startTime),
  130.             endTime: (isoToLocalTime(daySchedule.times[0].endTime) === '00:00' ? '23:59' : isoToLocalTime(daySchedule.times[0].endTime))
  131.         }] : [];
  132.     };
  133.     return weekDays.reduce((schedules: { [k: string]: any }, dayOfWeek: string) => {
  134.         const newSchedule = schedules;
  135.  
  136.         newSchedule[dayOfWeek] = findDay(dayOfWeek);
  137.         return newSchedule;
  138.     }, weeklySchedules);
  139. };
  140.  
  141. const generatePricing = (pricing: [Object]) => {
  142.     const price = new Map();
  143.     let ref = 1;
  144.     const xah_map_to_obj = ((aMap: any) => {
  145.         const obj: any = {};
  146.         aMap.forEach((v: any, k: any) => {
  147.             obj[k] = v
  148.         });
  149.         return obj;
  150.     });
  151.  
  152.  
  153.     pricing.forEach((offer: any) => {
  154.         price.set('p' + ('00' + ref++).slice(-3), {
  155.             label: {
  156.                 fr: offer.descriptions[0].title.length >= 2 ? offer.descriptions[0].title : 'MISSING',
  157.             },
  158.             minimumNumberOfParticipants: offer.minVisitor,
  159.             amountInCents: offer.price >= 100 ? offer.price : 4242
  160.         });
  161.     });
  162.     return xah_map_to_obj(price);
  163. };
  164.  
  165. const generateFullDescription = (descriptions: any, pricing: any) => {
  166.     let description: string = '';
  167.     let priceDescription: string = '';
  168.  
  169.     for (let paragraph of descriptions) {
  170.         if (paragraph.title && paragraph.title !== 'OTHER' && paragraph.description)
  171.             description += '### ' + paragraph.title + '\n' + paragraph.description + '\n\n';
  172.     }
  173.     pricing.forEach((offer: any) => {
  174.         if (offer.descriptions[0].description && offer.descriptions[0].title) {
  175.             priceDescription === '' ? priceDescription += '### Detail des tarifs\n' : 0;
  176.             priceDescription += offer.descriptions[0].title + ': ' + offer.descriptions[0].description + '\n\n';
  177.         }
  178.     });
  179.     if (description.length + priceDescription.length >= 10000) {
  180.         return description.substring(0, 9990 - priceDescription.length) + '...\n' + priceDescription;
  181.     }
  182.     return description + priceDescription;
  183. };
  184.  
  185. const updateDraftedExperienceVersions = async (experience: IExperience) => {
  186.     const xpv = await makePostRequest(API_URL + '/experience-versions/draft', experience);
  187.     let errorCheck;
  188.  
  189.     if (!xpv || !xpv.id)
  190.         return console.log('ERROR WHILE DRAFTING THE EXPV: ', xpv.message);
  191.     experience.experienceVersionId = xpv.id;
  192.     errorCheck = await makePostRequest(API_URL + '/experience-versions/update-pricing', experience);
  193.     errorCheck.message ? console.log('ERROR WHILE UPDATING PRICING: ', errorCheck.message, experience.pricing) : 0;
  194.     errorCheck = await makePostRequest(API_URL + '/experience-versions/update-opening-times', experience);
  195.     errorCheck.message ? console.log('ERROR WHILE UPDATING OPENING-TIME: ', errorCheck.message, experience.openingTimes) : 0;
  196.     errorCheck = await makePostRequest(API_URL + '/experience-versions/update-description', experience);
  197.     errorCheck.message ? console.log('ERROR WHILE UPDATING DESCRIPTION: ', errorCheck.message, experience.description) : 0;
  198.     errorCheck = await makePostRequest(API_URL + '/experience-versions/update-commencement-date', experience);
  199.     errorCheck.message ? console.log('ERROR WHILE UPDATING COMMENCEMENT-DATE: ', errorCheck.message, experience.commencementDate) : 0;
  200.     errorCheck = await makePostRequest(API_URL + '/experience-versions/update-picture-urls', experience);
  201.     errorCheck.message ? console.log('ERROR WHILE UPDATING PICTURES URL: ', errorCheck.message, experience.pictureUrls) : 0;
  202.     // AUTO VALIDATION TO COMMENT
  203.     // errorCheck = await makePostRequest(API_URL + '/experience-versions/approve', experience);
  204.     // errorCheck.message ? console.log('ERROR WHILE APPROVING EXPERIENCE: ', errorCheck.message, experience.pictureUrls) : 0;
  205.     return (xpv.id);
  206. };
  207.  
  208. const generateScrappedExperiences = async (data: any, picturesMap: Map<string, [string]>, ids: Iids) => {
  209.     data.patriServices.forEach(async (service: any) => {
  210.         const pricing = generatePricing(service.offers);
  211.         const schedules = generateSchedule(service.schedules);
  212.         const description = generateFullDescription(data.descriptions, service.offers);
  213.         const experience: IExperience = {
  214.             hostId: ids.hostId,
  215.             landmarkId: ids.landmarkId,
  216.             title: {
  217.                 fr: data.descriptions.find((elem: any) => {
  218.                     return elem.type === 'TITLE';
  219.                 }).title
  220.             },
  221.             slug: {
  222.                 fr: slug(data.descriptions.find((elem: any) => {
  223.                     return elem.type === 'TITLE';
  224.                 }).title, {lower: true})
  225.             },
  226.             experienceFormat: {
  227.                 formatLabel: {
  228.                     fr: service.descriptions[0].title || 'no description available'
  229.                 },
  230.                 sessionFormat: 'SESSION'
  231.             },
  232.             description: {
  233.                 fr: description || service.descriptions[0].description || 'no description available'
  234.             },
  235.             openingTimes: schedules,
  236.             pictureUrls: ids.imagesIds,
  237.             weeklySessionPattern: {},
  238.             sessionParameters: {
  239.                 timeInMinutes: -1,
  240.                 maximumNumberOfParticipants: service.maxPerHour,
  241.                 durationType: 'UNAVAILABLE'
  242.             },
  243.             pricing: {
  244.                 prices: pricing,
  245.             },
  246.             commencementDate: LocalDate.parse(service.schedules[0].periods[0].start).atStartOfDay().toString() + ':00Z',
  247.             unavailabilityReason: 'COMING_SOON'
  248.         };
  249.         const response = await makePostRequest(API_URL + '/experiences/create', {
  250.             hostId: experience.hostId,
  251.             title: experience.title,
  252.             slug: experience.slug
  253.         });
  254.  
  255.         if (!response || !response.id)
  256.             return console.log('ERROR WHILE CREATING THE EXPERIENCE: ', response.message);
  257.         experience.experienceId = response.id;
  258.         picturesMap.set(ids.hostId, ids.imagesIds);
  259.         await updateDraftedExperienceVersions(experience);
  260.     });
  261.     return picturesMap;
  262. };
  263.  
  264. const generateScrappedLandmarks = async (data: any, picturesMap: Map<string, [string]>, hostId: string) => {
  265.     const landmark: ILandmark = {
  266.         address: {
  267.             locality: data.location.city,
  268.             country: data.location.country,
  269.             longitude: data.location.longitude,
  270.             latitude: data.location.latitude
  271.         },
  272.         name: {
  273.             fr: data.descriptions.find((elem: any) => {
  274.                 return elem.type === 'TITLE';
  275.             }).title,
  276.         }
  277.     };
  278.     let response = await makePostRequest(API_URL + '/landmarks/create', landmark);
  279.     const images: any = [];
  280.  
  281.     if (!response || !response.id)
  282.         return console.log('ERROR WHILE CREATING THE LANDMARK: ', response.message);
  283.     data.images.forEach((img: any) => {
  284.         images.push(img.patriPlaceId + '-' + img.id + '.jpg');
  285.     });
  286.     await generateScrappedExperiences(data, picturesMap, {hostId, landmarkId: response.id, imagesIds: images});
  287.     return;
  288. };
  289.  
  290. const generateScrappedUsers = async (data: Array<any>) => {
  291.     let picturesMap: Map<string, [string]> = new Map();
  292.     let i: number = 0;
  293.  
  294.     for (let elem of data) {
  295.         const user: IUser = {
  296.             email: 'lucas+' + i + encodeURI(elem.url.replace("'", '')) + '@dartagnans.fr',
  297.             password: 'uWNa5g89',
  298.             firstName: 'scrap',
  299.             lastName: 'ping',
  300.             dateOfBirth: '1990-02-12'
  301.         };
  302.         const response = await makePostRequest(API_URL + '/users/sign-up', user);
  303.  
  304.         if (!response || !response.id)
  305.             return console.log('ERROR WHILE CREATING THE USER: ', response.message);
  306.         const host = await makePostRequest(API_URL + '/hosts/submit', {
  307.             userId: response.id,
  308.             nationality: "FRA",
  309.             countryOfResidence: "FRA"
  310.         });
  311.         if (!host || !host.id)
  312.             return console.log('ERROR WHILE GIVING THE HOST ROLE: ', host.message);
  313.         await generateScrappedLandmarks(elem, picturesMap, host.id);
  314.         i++;
  315.         // if (i > 6)
  316.         //     break ;
  317.         // await sleep(3000);
  318.     }
  319.     fs.writeFile("./picturesMap.js", JSON.stringify([...picturesMap]), (err) => {
  320.         if (err) {
  321.             return console.log(err);
  322.         }
  323.     });
  324.     console.log('EVERYTHING WAS INSERTED');
  325. };
  326.  
  327. // dump informations: 70 experiences, 42 experiencesVersions, 76 landmarks, 271 users, 13 hosts
  328. // +612 exoeriences, +612 experiences, +264 landmarks, +264 users, +264 hosts
  329.  
  330. const checkExpectedResults = async (data: any) => {
  331.     const res: any = [];
  332.     const result: any = [];
  333.  
  334.     data.forEach((elem: any) => {
  335.         if (elem.patriServices) {
  336.             elem.patriServices.forEach((offer: any) => {
  337.                 res.push(offer);
  338.             });
  339.             result.push(elem);
  340.         }
  341.     });
  342.     await generateScrappedUsers(data);
  343.     console.log('Source file contain: ' + data.length + ' entries');
  344.     console.log('There is: ' + result.length + ' patriServices in all theses entries');
  345.     console.log('Total: ' + res.length + ' experiences versions should have been inserted');
  346.     return;
  347. };
  348.  
  349. checkExpectedResults(scrap);
  350. // generateScrappedUsers(scrap);
Advertisement
Add Comment
Please, Sign In to add comment