streetmerchant/src/store/fetch-links.ts

import {Link, Series, Store} from './model';
import {Print, logger} from '../logger';
import {Browser} from 'puppeteer';
import cheerio from 'cheerio';
import {filterSeries} from './filter';
import {usingResponse} from '../util';

function addNewLinks(store: Store, links: Link[], series: Series) {
	if (links.length === 0) {
		logger.debug(Print.message('NO STORE LINKS FOUND', series, store, true));

		return;
	}

	const existingUrls = new Set(store.links.map(link => link.url));
	const newLinks = links.filter(link => !existingUrls.has(link.url));

	if (newLinks.length === 0) {
		return;
	}

	logger.debug(Print.message(`FOUND ${newLinks.length} STORE LINKS`, series, store, true));
	logger.debug(JSON.stringify(newLinks, null, 2));

	store.links = store.links.concat(newLinks);
}

export async function fetchLinks(store: Store, browser: Browser) {
	if (!store.linksBuilder) {
		return;
	}

	const promises: Array<Promise<void>> = [];

	for (let {series, url} of store.linksBuilder.urls) {
		if (!filterSeries(series)) {
			continue;
		}

		logger.debug(Print.message('DETECTING STORE LINKS', series, store, true));

		if (!Array.isArray(url)) {
			url = [url];
		}

		url.map(x => promises.push(usingResponse(browser, x, async response => {
			const text = await response?.text();

			if (!text) {
				logger.error(Print.message('NO RESPONSE', series, store, true));
				return;
			}

			const docElement = cheerio.load(text).root();
			const links = store.linksBuilder!.builder(docElement, series);

			addNewLinks(store, links, series);
		})));
	}

	await Promise.all(promises);
}