opelly/streetmerchant
1
0
Fork 0
mirror of https://github.com/opelly27/streetmerchant.git synced 2026-05-20 05:17:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: fix metadata in logging, lint prettier, typed Models (#747)

Browse Source
This commit is contained in:
Jef LeCompte
2020-11-10 10:38:14 -05:00
committed by GitHub
parent 0bf9ba2e18
commit 318e626e78
78 changed files with 2416 additions and 1021 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/store/fetch-links.ts
+19 -13
View File
@@ -12,14 +12,16 @@ function addNewLinks(store: Store, links: Link[], series: Series) {
return;
}
const existingUrls = new Set(store.links.map(link => link.url));
const newLinks = links.filter(link => !existingUrls.has(link.url));
const existingUrls = new Set(store.links.map((link) => link.url));
const newLinks = links.filter((link) => !existingUrls.has(link.url));
if (newLinks.length === 0) {
return;
}
logger.debug(Print.message(`FOUND ${newLinks.length} STORE LINKS`, series, store, true));
logger.debug(
Print.message(`FOUND ${newLinks.length} STORE LINKS`, series, store, true)
);
logger.debug(JSON.stringify(newLinks, null, 2));
store.links = store.links.concat(newLinks);
@@ -43,19 +45,23 @@ export async function fetchLinks(store: Store, browser: Browser) {
url = [url];
}
url.map(x => promises.push(usingResponse(browser, x, async response => {
const text = await response?.text();
url.map((x) =>
promises.push(
usingResponse(browser, x, async (response) => {
const text = await response?.text();
if (!text) {
logger.error(Print.message('NO RESPONSE', series, store, true));
return;
}
if (!text) {
logger.error(Print.message('NO RESPONSE', series, store, true));
return;
}
const docElement = cheerio.load(text).root();
const links = store.linksBuilder!.builder(docElement, series);
const docElement = cheerio.load(text).root();
const links = store.linksBuilder!.builder(docElement, series);
addNewLinks(store, links, series);
})));
addNewLinks(store, links, series);
})
)
);
}
await Promise.all(promises);