Webz.io Google Apps Script for Newsletter

/*
 * WEEKLY NEWS NEWSLETTER
 * Webz.io News Search API + Google Apps Script
 *
 * Change only the CONFIG section below.
 */

const CONFIG = {
  QUERY: 'Supply chain disruptions affecting European car manufacturers',

  DAYS_BACK: 7,
  RESULTS_TO_FETCH: 30,
  STORIES_TO_INCLUDE: 8,
  MAX_STORIES_PER_DOMAIN: 2,

  FILTERS: {
    language: ['english'],
    country: [],
    category: ['Economy, Business and Finance'],
    sentiment: [],
    domain: [],
    exclude_domain: []
  },

  NEWSLETTER_TITLE: 'Weekly Supply Chain Brief',
  SENDER_NAME: 'Weekly Supply Chain Brief',

  TEST_RECIPIENT: '[email protected]',

  RECIPIENTS: [
    '[email protected]'
  ]
};

const WEBZ_ENDPOINT = 'https://api.webz.io/api/news/context';


function testNewsletter() {
  runNewsletter([CONFIG.TEST_RECIPIENT]);
}


function sendWeeklyNewsletter() {
  runNewsletter(CONFIG.RECIPIENTS);
}


function runNewsletter(recipients) {
  validateConfiguration(recipients);

  const data = searchWebz();
  const stories = selectStories(data.results || []);

  if (stories.length === 0) {
    console.log('No matching articles were found.');
    return;
  }

  const dateRange = getNewsletterDateRange();

  const subject =
    CONFIG.NEWSLETTER_TITLE +
    ' - ' +
    dateRange.start +
    ' to ' +
    dateRange.end;

  const htmlBody = buildHtmlNewsletter(stories, dateRange);
  const textBody = buildTextNewsletter(stories, dateRange);

  const remainingQuota = MailApp.getRemainingDailyQuota();

  if (remainingQuota < recipients.length) {
    throw new Error(
      'Not enough email quota remaining. Need ' +
      recipients.length +
      ', but only ' +
      remainingQuota +
      ' recipients remain today.'
    );
  }

  recipients.forEach(function(email) {
    MailApp.sendEmail({
      to: email,
      subject: subject,
      body: textBody,
      htmlBody: htmlBody,
      name: CONFIG.SENDER_NAME
    });
  });

  console.log(
    'Newsletter sent to ' +
    recipients.length +
    ' recipient(s).'
  );

  console.log(
    'Stories included: ' +
    stories.length
  );

  if (data.credits_used !== undefined) {
    console.log(
      'Webz.io credits used: ' +
      data.credits_used
    );
  }
}


function searchWebz() {
  const apiKey = PropertiesService
    .getScriptProperties()
    .getProperty('WEBZ_API_KEY');

  if (!apiKey) {
    throw new Error(
      'WEBZ_API_KEY was not found. ' +
      'Add it under Project Settings > Script Properties.'
    );
  }

  const payload = {
    query: CONFIG.QUERY,
    k: CONFIG.RESULTS_TO_FETCH,
    filters: buildWebzFilters()
  };

  console.log(
    'Webz.io request: ' +
    JSON.stringify(payload)
  );

  const response = UrlFetchApp.fetch(
    WEBZ_ENDPOINT,
    {
      method: 'post',

      contentType: 'application/json',

      headers: {
        Authorization: 'Bearer ' + apiKey
      },

      payload: JSON.stringify(payload),

      muteHttpExceptions: true
    }
  );

  const status = response.getResponseCode();
  const responseText = response.getContentText();

  if (status < 200 || status >= 300) {
    throw new Error(
      'Webz.io returned HTTP ' +
      status +
      '\n\n' +
      responseText
    );
  }

  try {
    return JSON.parse(responseText);
  } catch (error) {
    throw new Error(
      'Webz.io returned invalid JSON: ' +
      error.message
    );
  }
}


function buildWebzFilters() {
  const filters = {
    published_from: getPublishedFrom()
  };

  addFilterIfNotEmpty(
    filters,
    'language',
    CONFIG.FILTERS.language
  );

  addFilterIfNotEmpty(
    filters,
    'country',
    CONFIG.FILTERS.country
  );

  addFilterIfNotEmpty(
    filters,
    'category',
    CONFIG.FILTERS.category
  );

  addFilterIfNotEmpty(
    filters,
    'sentiment',
    CONFIG.FILTERS.sentiment
  );

  addFilterIfNotEmpty(
    filters,
    'domain',
    CONFIG.FILTERS.domain
  );

  addFilterIfNotEmpty(
    filters,
    'exclude_domain',
    CONFIG.FILTERS.exclude_domain
  );

  return filters;
}


function addFilterIfNotEmpty(filters, name, values) {
  if (
    Array.isArray(values) &&
    values.length > 0
  ) {
    filters[name] = values;
  }
}


function getPublishedFrom() {
  const date = new Date();

  date.setUTCDate(
    date.getUTCDate() -
    CONFIG.DAYS_BACK
  );

  return date.toISOString();
}


function selectStories(results) {
  const selected = [];
  const seenUrls = new Set();
  const domainCounts = {};

  for (let i = 0; i < results.length; i++) {
    const result = results[i] || {};
    const article = result.article || {};
    const metadata = result.metadata || {};

    if (
      !article.title ||
      !article.url
    ) {
      continue;
    }

    if (
      seenUrls.has(article.url)
    ) {
      continue;
    }

    const domain = String(
      metadata.domain ||
      getDomainFromUrl(article.url) ||
      ''
    ).toLowerCase();

    if (
      domain &&
      (domainCounts[domain] || 0) >=
      CONFIG.MAX_STORIES_PER_DOMAIN
    ) {
      continue;
    }

    seenUrls.add(article.url);

    if (domain) {
      domainCounts[domain] =
        (domainCounts[domain] || 0) + 1;
    }

    selected.push(result);

    if (
      selected.length >=
      CONFIG.STORIES_TO_INCLUDE
    ) {
      break;
    }
  }

  return selected;
}


function buildHtmlNewsletter(stories, dateRange) {
  let storiesHtml = '';

  stories.forEach(function(result) {
    const article = result.article || {};
    const metadata = result.metadata || {};

    const source =
      metadata.domain ||
      getDomainFromUrl(article.url) ||
      'News source';

    const published =
      formatPublishedDate(
        article.published_at
      );

    const summary =
      getStorySummary(result);

    storiesHtml +=
      '<div style="' +
        'margin:0 0 28px 0;' +
        'padding:0 0 24px 0;' +
        'border-bottom:1px solid #e5e5e5;' +
      '">' +

        '<div style="' +
          'color:#777;' +
          'font-size:13px;' +
          'margin-bottom:6px;' +
        '">' +

          escapeHtml(source) +

          (
            published
              ? ' &middot; ' +
                escapeHtml(published)
              : ''
          ) +

        '</div>' +

        '<div style="' +
          'font-size:19px;' +
          'font-weight:bold;' +
          'line-height:1.4;' +
          'margin-bottom:8px;' +
        '">' +

          escapeHtml(article.title) +

        '</div>' +

        '<div style="' +
          'font-size:15px;' +
          'line-height:1.6;' +
          'margin-bottom:10px;' +
        '">' +

          escapeHtml(summary) +

        '</div>' +

        '<a href="' +
          escapeHtml(article.url) +
          '" style="' +
          'font-size:14px;' +
          'font-weight:bold;' +
        '">' +

          'Read the article &rarr;' +

        '</a>' +

      '</div>';
  });

  return (
    '<div style="' +
      'max-width:650px;' +
      'margin:0 auto;' +
      'font-family:Arial,Helvetica,sans-serif;' +
      'color:#222;' +
    '">' +

      '<div style="' +
        'font-size:28px;' +
        'font-weight:bold;' +
        'margin-bottom:5px;' +
      '">' +

        escapeHtml(
          CONFIG.NEWSLETTER_TITLE
        ) +

      '</div>' +

      '<div style="' +
        'color:#777;' +
        'font-size:14px;' +
        'margin-bottom:28px;' +
      '">' +

        escapeHtml(dateRange.start) +

        ' - ' +

        escapeHtml(dateRange.end) +

      '</div>' +

      '<div style="' +
        'font-size:16px;' +
        'line-height:1.6;' +
        'margin-bottom:30px;' +
      '">' +

        'Weekly news about <strong>' +

        escapeHtml(CONFIG.QUERY) +

        '</strong>' +

      '</div>' +

      storiesHtml +

      '<div style="' +
        'color:#999;' +
        'font-size:12px;' +
        'margin-top:30px;' +
      '">' +

        'News retrieved using the Webz.io News Search API.' +

      '</div>' +

    '</div>'
  );
}


function buildTextNewsletter(stories, dateRange) {
  let output =
    CONFIG.NEWSLETTER_TITLE +
    '\n' +
    dateRange.start +
    ' - ' +
    dateRange.end +
    '\n\n' +
    'Weekly news about: ' +
    CONFIG.QUERY +
    '\n\n';

  stories.forEach(
    function(result, index) {
      const article =
        result.article || {};

      const metadata =
        result.metadata || {};

      const source =
        metadata.domain ||
        getDomainFromUrl(
          article.url
        ) ||
        'News source';

      output +=
        (index + 1) +
        '. ' +
        article.title +
        '\n' +

        source +
        '\n' +

        getStorySummary(result) +
        '\n' +

        article.url +
        '\n\n';
    }
  );

  return output;
}


function getStorySummary(result) {
  const article =
    result.article || {};

  const chunk =
    result.chunk || {};

  const text =
    article.summary ||
    chunk.text ||
    '';

  return truncateText(
    text,
    500
  );
}


function truncateText(text, maxLength) {
  text = String(text || '')
    .replace(/\s+/g, ' ')
    .trim();

  if (
    text.length <= maxLength
  ) {
    return text;
  }

  return (
    text
      .substring(
        0,
        maxLength - 3
      )
      .trim() +
    '...'
  );
}


function formatPublishedDate(publishedAt) {
  if (!publishedAt) {
    return '';
  }

  const date =
    new Date(publishedAt);

  if (
    isNaN(date.getTime())
  ) {
    return '';
  }

  return Utilities.formatDate(
    date,
    Session.getScriptTimeZone(),
    'MMM d, yyyy'
  );
}


function getNewsletterDateRange() {
  const end = new Date();

  const start =
    new Date(
      end.getTime() -
      CONFIG.DAYS_BACK *
      24 *
      60 *
      60 *
      1000
    );

  return {
    start: Utilities.formatDate(
      start,
      Session.getScriptTimeZone(),
      'MMM d, yyyy'
    ),

    end: Utilities.formatDate(
      end,
      Session.getScriptTimeZone(),
      'MMM d, yyyy'
    )
  };
}


function getDomainFromUrl(url) {
  return String(url || '')
    .replace(/^https?:\/\//i, '')
    .replace(/^www\./i, '')
    .split('/')[0]
    .split('?')[0]
    .split('#')[0];
}


function escapeHtml(value) {
  return String(value || '')
    .replace(/&/g, '&amp;')
    .replace(/</g, '&lt;')
    .replace(/>/g, '&gt;')
    .replace(/"/g, '&quot;')
    .replace(/'/g, '&#039;');
}


function validateConfiguration(recipients) {
  if (
    !CONFIG.QUERY ||
    !CONFIG.QUERY.trim()
  ) {
    throw new Error(
      'CONFIG.QUERY is empty.'
    );
  }

  if (
    !Number.isInteger(
      CONFIG.DAYS_BACK
    ) ||
    CONFIG.DAYS_BACK < 1 ||
    CONFIG.DAYS_BACK > 30
  ) {
    throw new Error(
      'DAYS_BACK must be an integer between 1 and 30.'
    );
  }

  if (
    !Number.isInteger(
      CONFIG.RESULTS_TO_FETCH
    ) ||
    CONFIG.RESULTS_TO_FETCH < 1 ||
    CONFIG.RESULTS_TO_FETCH > 50
  ) {
    throw new Error(
      'RESULTS_TO_FETCH must be an integer between 1 and 50.'
    );
  }

  if (
    !Number.isInteger(
      CONFIG.STORIES_TO_INCLUDE
    ) ||
    CONFIG.STORIES_TO_INCLUDE < 1
  ) {
    throw new Error(
      'STORIES_TO_INCLUDE must be a positive integer.'
    );
  }

  if (
    !Number.isInteger(
      CONFIG.MAX_STORIES_PER_DOMAIN
    ) ||
    CONFIG.MAX_STORIES_PER_DOMAIN < 1
  ) {
    throw new Error(
      'MAX_STORIES_PER_DOMAIN must be a positive integer.'
    );
  }

  if (
    !Array.isArray(recipients) ||
    recipients.length === 0
  ) {
    throw new Error(
      'No newsletter recipients were configured.'
    );
  }

  recipients.forEach(
    function(email) {
      if (
        !email ||
        !/^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(
          email
        )
      ) {
        throw new Error(
          'Invalid recipient: ' +
          email
        );
      }
    }
  );
}


function createWeeklyTrigger() {
  const triggers =
    ScriptApp.getProjectTriggers();

  triggers.forEach(
    function(trigger) {
      if (
        trigger.getHandlerFunction() ===
        'sendWeeklyNewsletter'
      ) {
        ScriptApp.deleteTrigger(
          trigger
        );
      }
    }
  );

  ScriptApp
    .newTrigger(
      'sendWeeklyNewsletter'
    )
    .timeBased()
    .everyWeeks(1)
    .onWeekDay(
      ScriptApp.WeekDay.MONDAY
    )
    .atHour(9)
    .create();

  console.log(
    'Weekly newsletter trigger created.'
  );
}