Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import scrap = require('./scrapping_data');
- import "isomorphic-fetch"
- import {DateTimeFormatter, LocalDate, LocalDateTime} from 'js-joda';
- import fs = require('fs');
- const slug = require('slug');
- const API_URL = 'https://dartngo-api-production.herokuapp.com';
- // const API_TOKEN = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6eyJfaWQiOiJ1c3JfMjVkNTI1MDYtZTlmYy00NDVhLTkxNjQtMGZhN2FlNGMwZWYyIn0sImlhdCI6MTU0NDYwNzcyNCwiZXhwIjoxNTQ0ODY2OTI0fQ._CckpVxCNED2AdUJ6ECBdTXOo3nw-8nCu6Xkq9jUCB0';
- interface IUser extends Object {
- email: string;
- password: string;
- firstName: string;
- lastName: string;
- dateOfBirth: string;
- }
- interface ILandmark extends Object {
- address: {
- streetNumber?: string,
- route?: string,
- locality: string,
- administrativeAreaLevel1?: string,
- administrativeAreaLevel2?: string,
- country: string,
- postalCode?: string
- longitude: number,
- latitude: number
- };
- name: {
- fr: string,
- };
- }
- interface IExperience extends Object {
- id?: string;
- experienceId?: string;
- experienceVersionId?: string;
- hostId?: string;
- landmarkId?: string;
- title: {
- fr: string
- },
- slug: {
- fr: string
- },
- publicationStatus?: string;
- draftDate?: string;
- experienceFormat: {
- formatLabel: {
- fr: string,
- },
- sessionFormat?: string
- };
- timeZoneId?: string,
- defaultLanguage?: string;
- expiryDate?: string;
- supersededExperienceVersionId?: string;
- supersedingExperienceVersionId?: string;
- description: {
- fr: string
- };
- openingTimes: Object;
- pictureUrls?: [string];
- weeklySessionPattern?: any;
- sessionParameters: {
- timeInMinutes: Number,
- maximumNumberOfParticipants: Number,
- durationType?: string
- };
- pricing: {
- prices: Map<string, Object>,
- extras?: [{
- extraRef: string,
- label: [
- {
- language: string,
- value: string
- }
- ],
- amountInCents: Number
- }]
- };
- appropriateVisitorClasses?: [string];
- travelInformation?: [
- {
- language: string,
- value: string
- }
- ];
- commencementDate: string;
- unavailabilityReason?: string;
- }
- interface Iids extends Object {
- hostId: string,
- landmarkId: string,
- imagesIds: [string]
- }
- // const sleep = async (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
- const isoToLocalTime = (isoDate: string) => {
- const d = LocalDateTime.parse(isoDate);
- return d.format((DateTimeFormatter.ofPattern('HH:mm')));
- };
- const makePostRequest = async (url: string, data: any) => {
- return fetch(url, {
- method: 'POST', // or 'PUT'
- body: JSON.stringify(data), // data can be `string` or {object}!
- headers: {
- 'Content-Type': 'application/json',
- 'Authorization': 'Token eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6eyJfaWQiOiJ1c3JfMjVkNTI1MDYtZTlmYy00NDVhLTkxNjQtMGZhN2FlNGMwZWYyIn0sImlhdCI6MTU0NDYwNzcyNCwiZXhwIjoxNTQ0ODY2OTI0fQ._CckpVxCNED2AdUJ6ECBdTXOo3nw-8nCu6Xkq9jUCB0'
- }
- })
- .then(res => res.json())
- .catch(error => error);
- };
- const generateSchedule = (schedule: any) => {
- const data = schedule[0].periods[0].defaults;
- const weekDays: string[] = ["monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday"];
- const weeklySchedules: { [k: string]: any } = {};
- const findDay = (day: string) => {
- const daySchedule = data.find((elem: any) => elem.dayOfTheWeek === day.toUpperCase() && !!elem.times && !!elem.times[0]);
- return !!daySchedule ? [{
- startTime: isoToLocalTime(daySchedule.times[0].startTime),
- endTime: (isoToLocalTime(daySchedule.times[0].endTime) === '00:00' ? '23:59' : isoToLocalTime(daySchedule.times[0].endTime))
- }] : [];
- };
- return weekDays.reduce((schedules: { [k: string]: any }, dayOfWeek: string) => {
- const newSchedule = schedules;
- newSchedule[dayOfWeek] = findDay(dayOfWeek);
- return newSchedule;
- }, weeklySchedules);
- };
- const generatePricing = (pricing: [Object]) => {
- const price = new Map();
- let ref = 1;
- const xah_map_to_obj = ((aMap: any) => {
- const obj: any = {};
- aMap.forEach((v: any, k: any) => {
- obj[k] = v
- });
- return obj;
- });
- pricing.forEach((offer: any) => {
- price.set('p' + ('00' + ref++).slice(-3), {
- label: {
- fr: offer.descriptions[0].title.length >= 2 ? offer.descriptions[0].title : 'MISSING',
- },
- minimumNumberOfParticipants: offer.minVisitor,
- amountInCents: offer.price >= 100 ? offer.price : 4242
- });
- });
- return xah_map_to_obj(price);
- };
- const generateFullDescription = (descriptions: any, pricing: any) => {
- let description: string = '';
- let priceDescription: string = '';
- for (let paragraph of descriptions) {
- if (paragraph.title && paragraph.title !== 'OTHER' && paragraph.description)
- description += '### ' + paragraph.title + '\n' + paragraph.description + '\n\n';
- }
- pricing.forEach((offer: any) => {
- if (offer.descriptions[0].description && offer.descriptions[0].title) {
- priceDescription === '' ? priceDescription += '### Detail des tarifs\n' : 0;
- priceDescription += offer.descriptions[0].title + ': ' + offer.descriptions[0].description + '\n\n';
- }
- });
- if (description.length + priceDescription.length >= 10000) {
- return description.substring(0, 9990 - priceDescription.length) + '...\n' + priceDescription;
- }
- return description + priceDescription;
- };
- const updateDraftedExperienceVersions = async (experience: IExperience) => {
- const xpv = await makePostRequest(API_URL + '/experience-versions/draft', experience);
- let errorCheck;
- if (!xpv || !xpv.id)
- return console.log('ERROR WHILE DRAFTING THE EXPV: ', xpv.message);
- experience.experienceVersionId = xpv.id;
- errorCheck = await makePostRequest(API_URL + '/experience-versions/update-pricing', experience);
- errorCheck.message ? console.log('ERROR WHILE UPDATING PRICING: ', errorCheck.message, experience.pricing) : 0;
- errorCheck = await makePostRequest(API_URL + '/experience-versions/update-opening-times', experience);
- errorCheck.message ? console.log('ERROR WHILE UPDATING OPENING-TIME: ', errorCheck.message, experience.openingTimes) : 0;
- errorCheck = await makePostRequest(API_URL + '/experience-versions/update-description', experience);
- errorCheck.message ? console.log('ERROR WHILE UPDATING DESCRIPTION: ', errorCheck.message, experience.description) : 0;
- errorCheck = await makePostRequest(API_URL + '/experience-versions/update-commencement-date', experience);
- errorCheck.message ? console.log('ERROR WHILE UPDATING COMMENCEMENT-DATE: ', errorCheck.message, experience.commencementDate) : 0;
- errorCheck = await makePostRequest(API_URL + '/experience-versions/update-picture-urls', experience);
- errorCheck.message ? console.log('ERROR WHILE UPDATING PICTURES URL: ', errorCheck.message, experience.pictureUrls) : 0;
- // AUTO VALIDATION TO COMMENT
- // errorCheck = await makePostRequest(API_URL + '/experience-versions/approve', experience);
- // errorCheck.message ? console.log('ERROR WHILE APPROVING EXPERIENCE: ', errorCheck.message, experience.pictureUrls) : 0;
- return (xpv.id);
- };
- const generateScrappedExperiences = async (data: any, picturesMap: Map<string, [string]>, ids: Iids) => {
- data.patriServices.forEach(async (service: any) => {
- const pricing = generatePricing(service.offers);
- const schedules = generateSchedule(service.schedules);
- const description = generateFullDescription(data.descriptions, service.offers);
- const experience: IExperience = {
- hostId: ids.hostId,
- landmarkId: ids.landmarkId,
- title: {
- fr: data.descriptions.find((elem: any) => {
- return elem.type === 'TITLE';
- }).title
- },
- slug: {
- fr: slug(data.descriptions.find((elem: any) => {
- return elem.type === 'TITLE';
- }).title, {lower: true})
- },
- experienceFormat: {
- formatLabel: {
- fr: service.descriptions[0].title || 'no description available'
- },
- sessionFormat: 'SESSION'
- },
- description: {
- fr: description || service.descriptions[0].description || 'no description available'
- },
- openingTimes: schedules,
- pictureUrls: ids.imagesIds,
- weeklySessionPattern: {},
- sessionParameters: {
- timeInMinutes: -1,
- maximumNumberOfParticipants: service.maxPerHour,
- durationType: 'UNAVAILABLE'
- },
- pricing: {
- prices: pricing,
- },
- commencementDate: LocalDate.parse(service.schedules[0].periods[0].start).atStartOfDay().toString() + ':00Z',
- unavailabilityReason: 'COMING_SOON'
- };
- const response = await makePostRequest(API_URL + '/experiences/create', {
- hostId: experience.hostId,
- title: experience.title,
- slug: experience.slug
- });
- if (!response || !response.id)
- return console.log('ERROR WHILE CREATING THE EXPERIENCE: ', response.message);
- experience.experienceId = response.id;
- picturesMap.set(ids.hostId, ids.imagesIds);
- await updateDraftedExperienceVersions(experience);
- });
- return picturesMap;
- };
- const generateScrappedLandmarks = async (data: any, picturesMap: Map<string, [string]>, hostId: string) => {
- const landmark: ILandmark = {
- address: {
- locality: data.location.city,
- country: data.location.country,
- longitude: data.location.longitude,
- latitude: data.location.latitude
- },
- name: {
- fr: data.descriptions.find((elem: any) => {
- return elem.type === 'TITLE';
- }).title,
- }
- };
- let response = await makePostRequest(API_URL + '/landmarks/create', landmark);
- const images: any = [];
- if (!response || !response.id)
- return console.log('ERROR WHILE CREATING THE LANDMARK: ', response.message);
- data.images.forEach((img: any) => {
- images.push(img.patriPlaceId + '-' + img.id + '.jpg');
- });
- await generateScrappedExperiences(data, picturesMap, {hostId, landmarkId: response.id, imagesIds: images});
- return;
- };
- const generateScrappedUsers = async (data: Array<any>) => {
- let picturesMap: Map<string, [string]> = new Map();
- let i: number = 0;
- for (let elem of data) {
- const user: IUser = {
- email: 'lucas+' + i + encodeURI(elem.url.replace("'", '')) + '@dartagnans.fr',
- password: 'uWNa5g89',
- firstName: 'scrap',
- lastName: 'ping',
- dateOfBirth: '1990-02-12'
- };
- const response = await makePostRequest(API_URL + '/users/sign-up', user);
- if (!response || !response.id)
- return console.log('ERROR WHILE CREATING THE USER: ', response.message);
- const host = await makePostRequest(API_URL + '/hosts/submit', {
- userId: response.id,
- nationality: "FRA",
- countryOfResidence: "FRA"
- });
- if (!host || !host.id)
- return console.log('ERROR WHILE GIVING THE HOST ROLE: ', host.message);
- await generateScrappedLandmarks(elem, picturesMap, host.id);
- i++;
- // if (i > 6)
- // break ;
- // await sleep(3000);
- }
- fs.writeFile("./picturesMap.js", JSON.stringify([...picturesMap]), (err) => {
- if (err) {
- return console.log(err);
- }
- });
- console.log('EVERYTHING WAS INSERTED');
- };
- // dump informations: 70 experiences, 42 experiencesVersions, 76 landmarks, 271 users, 13 hosts
- // +612 exoeriences, +612 experiences, +264 landmarks, +264 users, +264 hosts
- const checkExpectedResults = async (data: any) => {
- const res: any = [];
- const result: any = [];
- data.forEach((elem: any) => {
- if (elem.patriServices) {
- elem.patriServices.forEach((offer: any) => {
- res.push(offer);
- });
- result.push(elem);
- }
- });
- await generateScrappedUsers(data);
- console.log('Source file contain: ' + data.length + ' entries');
- console.log('There is: ' + result.length + ' patriServices in all theses entries');
- console.log('Total: ' + res.length + ' experiences versions should have been inserted');
- return;
- };
- checkExpectedResults(scrap);
- // generateScrappedUsers(scrap);
Advertisement
Add Comment
Please, Sign In to add comment