feat: enhanced lookup behaviour (#270)

Co-authored-by: Jef LeCompte <jeffreylec@gmail.com>
This commit is contained in:
Andrew Mackrodt
2020-09-25 23:29:10 +01:00
committed by GitHub
parent 7ef9d935c6
commit b868d1a483
17 changed files with 789 additions and 71 deletions
+8
View File
@@ -2,6 +2,7 @@ import {Config} from './config';
import {Logger} from './logger';
import {Stores} from './store/model';
import {adBlocker} from './adblocker';
import {fetchLinks} from './store/fetch-links';
import {getSleepTime} from './util';
import puppeteer from 'puppeteer-extra';
import stealthPlugin from 'puppeteer-extra-plugin-stealth';
@@ -37,14 +38,21 @@ async function main() {
headless: Config.browser.isHeadless
});
const promises = [];
for (const store of Stores) {
Logger.debug(store.links);
if (store.setupAction !== undefined) {
store.setupAction(browser);
}
if (store.linksBuilder) {
promises.push(fetchLinks(store, browser));
}
setTimeout(tryLookupAndLoop, getSleepTime(), browser, store);
}
await Promise.all(promises);
}
/**