From 28947be9bc8981d7a45a5d0e69c18d039fcd9ed3 Mon Sep 17 00:00:00 2001 From: Jef LeCompte Date: Sun, 20 Sep 2020 11:02:57 -0400 Subject: [PATCH] fix: `rateLimitTimeout` not being defaulted (#106) refactor: `browser` and `store` config object Signed-off-by: Jef LeCompte --- src/config.ts | 30 ++++++++++++++---------------- src/index.ts | 6 +++--- src/store/lookup.ts | 6 +++--- src/store/model/index.ts | 3 +-- 4 files changed, 21 insertions(+), 24 deletions(-) diff --git a/src/config.ts b/src/config.ts index 1207810..f612a2f 100644 --- a/src/config.ts +++ b/src/config.ts @@ -3,6 +3,14 @@ import {config} from 'dotenv'; config({path: resolve(__dirname, '../.env')}); +const browser = { + isHeadless: process.env.HEADLESS ? process.env.HEADLESS === 'true' : true, + open: process.env.OPEN_BROWSER === 'true', + rateLimitTimeout: process.env.RATE_LIMIT_TIMEOUT ? Number(process.env.RATE_LIMIT_TIMEOUT) : 5000 +}; + +const logLevel = process.env.LOG_LEVEL ?? 'info'; + const notifications = { email: { username: process.env.EMAIL_USERNAME ?? '', @@ -45,25 +53,15 @@ const page = { userAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36' }; -const rateLimitTimeout = Number(process.env.RATE_LIMIT_TIMEOUT) ?? 5000; - -const stores = process.env.STORES ? process.env.STORES.split(',') : ['nvidia']; - -const openBrowser = process.env.OPEN_BROWSER === 'true'; - -const isHeadless = process.env.HEADLESS ? process.env.HEADLESS === 'true' : true; - -const showOnlyBrands = process.env.SHOW_ONLY_BRANDS ? process.env.SHOW_ONLY_BRANDS.split(',') : []; - -const logLevel = process.env.LOG_LEVEL ?? 'info'; +const store = { + showOnlyBrands: process.env.SHOW_ONLY_BRANDS ? process.env.SHOW_ONLY_BRANDS.split(',') : [], + stores: process.env.STORES ? process.env.STORES.split(',') : ['nvidia'] +}; export const Config = { - isHeadless, + browser, logLevel, notifications, - openBrowser, page, - rateLimitTimeout, - showOnlyBrands, - stores + store }; diff --git a/src/index.ts b/src/index.ts index 62ee12b..ed3304d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -16,7 +16,7 @@ puppeteer.use(adblockerPlugin({blockTrackers: true})); */ async function main() { const browser = await puppeteer.launch({ - headless: Config.isHeadless, + headless: Config.browser.isHeadless, defaultViewport: { height: Config.page.height, width: Config.page.width @@ -26,7 +26,7 @@ async function main() { const q = async.queue(async (store: Store, cb) => { setTimeout(async () => { try { - Logger.debug(`↗ Scraping Initialized - ${store.name}`); + Logger.debug(`↗ scraping initialized - ${store.name}`); await lookup(browser, store); } catch (error) { // Ignoring errors; more than likely due to rate limits @@ -35,7 +35,7 @@ async function main() { cb(); q.push(store); } - }, Config.rateLimitTimeout); + }, Config.browser.rateLimitTimeout); }, Stores.length); for (const store of Stores) { diff --git a/src/store/lookup.ts b/src/store/lookup.ts index aa4f46e..38fccde 100644 --- a/src/store/lookup.ts +++ b/src/store/lookup.ts @@ -12,11 +12,11 @@ import {includesLabels} from './includes-labels'; * @param brand The brand of the GPU */ function filterBrand(brand: string) { - if (Config.showOnlyBrands.length === 0) { + if (Config.store.showOnlyBrands.length === 0) { return true; } - return Config.showOnlyBrands.includes(brand); + return Config.store.showOnlyBrands.includes(brand); } /** @@ -70,7 +70,7 @@ export async function lookup(browser: Browser, store: Store) { const givenUrl = link.cartUrl ? link.cartUrl : link.url; - if (Config.openBrowser) { + if (Config.browser.open) { await open(givenUrl); } diff --git a/src/store/model/index.ts b/src/store/model/index.ts index 4728be9..fbee42e 100644 --- a/src/store/model/index.ts +++ b/src/store/model/index.ts @@ -1,4 +1,3 @@ - import {Amazon} from './amazon'; import {AmazonCa} from './amazon-ca'; import {Asus} from './asus'; @@ -25,7 +24,7 @@ const masterList = new Map([ const list = new Map(); -for (const name of Config.stores) { +for (const name of Config.store.stores) { list.set(name, masterList.get(name)); }