fix: rateLimitTimeout not being defaulted (#106)

refactor: `browser` and `store` config object
Signed-off-by: Jef LeCompte <jeffreylec@gmail.com>
This commit is contained in:
Jef LeCompte
2020-09-20 11:02:57 -04:00
committed by GitHub
parent 3de1f81eb1
commit 28947be9bc
4 changed files with 21 additions and 24 deletions
+3 -3
View File
@@ -16,7 +16,7 @@ puppeteer.use(adblockerPlugin({blockTrackers: true}));
*/
async function main() {
const browser = await puppeteer.launch({
headless: Config.isHeadless,
headless: Config.browser.isHeadless,
defaultViewport: {
height: Config.page.height,
width: Config.page.width
@@ -26,7 +26,7 @@ async function main() {
const q = async.queue<Store>(async (store: Store, cb) => {
setTimeout(async () => {
try {
Logger.debug(`Scraping Initialized - ${store.name}`);
Logger.debug(`scraping initialized - ${store.name}`);
await lookup(browser, store);
} catch (error) {
// Ignoring errors; more than likely due to rate limits
@@ -35,7 +35,7 @@ async function main() {
cb();
q.push(store);
}
}, Config.rateLimitTimeout);
}, Config.browser.rateLimitTimeout);
}, Stores.length);
for (const store of Stores) {