Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- const fetch = require('node-fetch');
- const fs = require('fs');
- const json = require("./partners.json");
- const createCsvWriter = require('csv-writer').createObjectCsvWriter;
- const LISTING_URL = 'https://partner-directory.adobe.io/v1/spp/listings';
- const DETAILS_URL = 'https://partner-directory.adobe.io/v1/spp/listing/';
- const JSON_FILE_NAME = __dirname + '/partners.json';
- const CSV_FILE_NAME = __dirname + '/partners.csv';
- const TOTAL_THREADS = 10;
- const fetchListingPage = pageNumber => {
- return new Promise(async (resolve, reject) => {
- console.log(`FETCHING PAGE ${ pageNumber }`);
- let url = LISTING_URL;
- if (pageNumber) {
- url += '?page=' + pageNumber
- }
- const rawResponse = await fetch(url, {
- headers: {
- 'X-Api-Key': 'partner_directory'
- }
- });
- const response = await rawResponse.json();
- resolve(response);
- });
- }
- let numberOfTimesDetailsLogged = 0;
- const fetchDetails = id => {
- return new Promise(async (resolve, reject) => {
- console.log(`${++numberOfTimesDetailsLogged}. FETCHING DETAILS OF ${ id }`);
- const url = DETAILS_URL + id;
- const rawResponse = await fetch(url, {
- headers: {
- 'X-Api-Key': 'partner_directory'
- }
- });
- const response = await rawResponse.json();
- resolve(response);
- });
- }
- const writeCSVFromJSON = async () => {
- console.log('Writing to CSV');
- const json = require(JSON_FILE_NAME);
- const csvData = [];
- const csvWriter = createCsvWriter({
- path: CSV_FILE_NAME,
- header: [
- {id: 'name', title: 'Name'},
- {id: 'level', title: 'Level'},
- {id: 'certifiedEmployees', title: 'Certified Employees'},
- {id: 'phone', title: 'Phone'},
- {id: 'website', title: 'Website'},
- {id: 'city', title: 'City'},
- {id: 'country', title: 'Country'},
- {id: 'countryId', title: 'Country ID'},
- {id: 'headquarter', title: 'Headquarter'},
- {id: 'address_name', title: 'Address Name'},
- {id: 'postalCode', title: 'Postal Code'},
- {id: 'regionId', title: 'Region ID'},
- {id: 'state', title: 'State'},
- {id: 'address1', title: 'Address 1'},
- {id: 'address2', title: 'Address 2'}
- ]
- });
- Object.values(json).forEach(company => {
- company.addresses.forEach(address => {
- csvData.push({
- 'name': company.companyInfo.name,
- 'level': company.companyInfo.level,
- 'certifiedEmployees': company.companyInfo.certifiedEmployees,
- 'phone': company.companyInfo.phone,
- 'website': company.companyInfo.website,
- 'city': address.city,
- 'country': address.country,
- 'countryId': address.countryId,
- 'headquarter': (address.isHeadquarter ? 'Yes': 'No'),
- 'address_name': address.name,
- 'postalCode': address.postalCode,
- 'regionId': address.regionId,
- 'state': address.state,
- 'address1': address.address1,
- 'address2': address.address2,
- })
- });
- });
- await csvWriter.writeRecords(csvData);
- console.log('DONE');
- }
- (async () => {
- let list = [];
- let totalItems;
- const finalList = {};
- let totalPages = null;
- let currentPage = 0;
- while (true) {
- const { listings = [], totalPages: pages = null, currentPage: current = null } = await fetchListingPage(currentPage);
- list = [...list, ...listings];
- if (pages) {
- totalPages = pages;
- }
- if (current) {
- currentPage = parseInt(current);
- }
- currentPage++;
- if (totalPages < currentPage) {
- break;
- }
- }
- totalItems = list.length;
- const runner = async () => {
- if (list.length) {
- const {id} = list.shift();
- finalList[id] = await fetchDetails(id);
- await runner();
- }
- }
- for (let i = 0; i < TOTAL_THREADS; i++) {
- runner();
- }
- const processId = setInterval(() => {
- if (Object.keys(finalList).length === totalItems) {
- clearInterval(processId);
- fs.writeFileSync(JSON_FILE_NAME, JSON.stringify(finalList, null, 2));
- console.log('DONE');
- console.log('JSON written to file ' + JSON_FILE_NAME);
- writeCSVFromJSON();
- }
- }, 1000);
- })();
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement