streetmerchant/src/store/fetch-links.ts

import {Link, Series, Store} from './model';
import {Print, logger} from '../logger';
import {Browser} from 'puppeteer';
import cheerio from 'cheerio';
import {filterSeries} from './filter';
import {usingResponse} from '../util';

function addNewLinks(store: Store, links: Link[], series: Series) {
  if (links.length === 0) {
    logger.debug(Print.message('NO STORE LINKS FOUND', series, store, true));

    return;
  }

  const existingUrls = new Set(store.links.map(link => link.url));
  const newLinks = links.filter(link => !existingUrls.has(link.url));

  if (newLinks.length === 0) {
    return;
  }

  logger.debug(
    Print.message(`FOUND ${newLinks.length} STORE LINKS`, series, store, true)
  );
  logger.debug(JSON.stringify(newLinks, null, 2));

  store.links = store.links.concat(newLinks);
}

export async function fetchLinks(store: Store, browser: Browser) {
  if (!store.linksBuilder) {
    return;
  }

  const promises: Array<Promise<void>> = [];

  // eslint-disable-next-line prefer-const
  for (let {series, url} of store.linksBuilder.urls) {
    if (!filterSeries(series)) {
      continue;
    }

    logger.debug(Print.message('DETECTING STORE LINKS', series, store, true));

    if (!Array.isArray(url)) {
      url = [url];
    }

    url.map(x =>
      promises.push(
        usingResponse(browser, x, async response => {
          const text = await response?.text();

          if (!text) {
            logger.error(Print.message('NO RESPONSE', series, store, true));
            return;
          }

          const docElement = cheerio.load(text).root();
          const links = store.linksBuilder!.builder(docElement, series);

          addNewLinks(store, links, series);
        })
      )
    );
  }

  await Promise.all(promises);
}