adobe_partners_scraper.js

const fetch = require('node-fetch');
const fs = require('fs');
const json = require("./partners.json");
const createCsvWriter = require('csv-writer').createObjectCsvWriter;

const LISTING_URL = 'https://partner-directory.adobe.io/v1/spp/listings';
const DETAILS_URL = 'https://partner-directory.adobe.io/v1/spp/listing/';
const JSON_FILE_NAME = __dirname + '/partners.json';
const CSV_FILE_NAME = __dirname + '/partners.csv';
const TOTAL_THREADS = 10;

const fetchListingPage = pageNumber => {
    return new Promise(async (resolve, reject) => {
        console.log(`FETCHING PAGE ${ pageNumber }`);
        let url = LISTING_URL;
        if (pageNumber) {
            url += '?page=' + pageNumber
        }
        const rawResponse = await fetch(url, {
            headers: {
                'X-Api-Key': 'partner_directory'
            }
        });
        const response = await rawResponse.json();
        resolve(response);
    });
}

let numberOfTimesDetailsLogged = 0;
const fetchDetails = id => {
    return new Promise(async (resolve, reject) => {
        console.log(`${++numberOfTimesDetailsLogged}. FETCHING DETAILS OF ${ id }`);
        const url = DETAILS_URL + id;
        const rawResponse = await fetch(url, {
            headers: {
                'X-Api-Key': 'partner_directory'
            }
        });
        const response = await rawResponse.json();
        resolve(response);
    });
}

const writeCSVFromJSON = async () => {
    console.log('Writing to CSV');
    const json = require(JSON_FILE_NAME);
    const csvData = [];
    const csvWriter = createCsvWriter({
        path: CSV_FILE_NAME,
        header: [
            {id: 'name', title: 'Name'},
            {id: 'level', title: 'Level'},
            {id: 'certifiedEmployees', title: 'Certified Employees'},
            {id: 'phone', title: 'Phone'},
            {id: 'website', title: 'Website'},
            {id: 'city', title: 'City'},
            {id: 'country', title: 'Country'},
            {id: 'countryId', title: 'Country ID'},
            {id: 'headquarter', title: 'Headquarter'},
            {id: 'address_name', title: 'Address Name'},
            {id: 'postalCode', title: 'Postal Code'},
            {id: 'regionId', title: 'Region ID'},
            {id: 'state', title: 'State'},
            {id: 'address1', title: 'Address 1'},
            {id: 'address2', title: 'Address 2'}
        ]
    });
    Object.values(json).forEach(company => {
        company.addresses.forEach(address => {
            csvData.push({
                'name': company.companyInfo.name,
                'level': company.companyInfo.level,
                'certifiedEmployees': company.companyInfo.certifiedEmployees,
                'phone': company.companyInfo.phone,
                'website': company.companyInfo.website,
                'city': address.city,
                'country': address.country,
                'countryId': address.countryId,
                'headquarter': (address.isHeadquarter ? 'Yes': 'No'),
                'address_name': address.name,
                'postalCode': address.postalCode,
                'regionId': address.regionId,
                'state': address.state,
                'address1': address.address1,
                'address2': address.address2,
            })
        });
    });
    await csvWriter.writeRecords(csvData);
    console.log('DONE');
}

(async () => {
    let list = [];
    let totalItems;
    const finalList = {};
    let totalPages = null;
    let currentPage = 0;
    while (true) {
        const { listings = [], totalPages: pages = null, currentPage: current = null } = await fetchListingPage(currentPage);
        list = [...list, ...listings];
        if (pages) {
            totalPages = pages;
        }
        if (current) {
            currentPage = parseInt(current);
        }
        currentPage++;
        if (totalPages < currentPage) {
            break;
        }
    }
    totalItems = list.length;

    const runner = async () => {
        if (list.length) {
            const {id} = list.shift();
            finalList[id] = await fetchDetails(id);
            await runner();
        }
    }

    for (let i = 0; i < TOTAL_THREADS; i++) {
        runner();
    }

    const processId = setInterval(() => {
        if (Object.keys(finalList).length === totalItems) {
            clearInterval(processId);
            fs.writeFileSync(JSON_FILE_NAME, JSON.stringify(finalList, null, 2));
            console.log('DONE');
            console.log('JSON written to file ' + JSON_FILE_NAME);
            writeCSVFromJSON();
        }
    }, 1000);
})();