fix: rateLimitTimeout not being defaulted (#106)

refactor: `browser` and `store` config object
Signed-off-by: Jef LeCompte <jeffreylec@gmail.com>
This commit is contained in:
Jef LeCompte
2020-09-20 11:02:57 -04:00
committed by GitHub
parent 3de1f81eb1
commit 28947be9bc
4 changed files with 21 additions and 24 deletions
+14 -16
View File
@@ -3,6 +3,14 @@ import {config} from 'dotenv';
config({path: resolve(__dirname, '../.env')});
const browser = {
isHeadless: process.env.HEADLESS ? process.env.HEADLESS === 'true' : true,
open: process.env.OPEN_BROWSER === 'true',
rateLimitTimeout: process.env.RATE_LIMIT_TIMEOUT ? Number(process.env.RATE_LIMIT_TIMEOUT) : 5000
};
const logLevel = process.env.LOG_LEVEL ?? 'info';
const notifications = {
email: {
username: process.env.EMAIL_USERNAME ?? '',
@@ -45,25 +53,15 @@ const page = {
userAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'
};
const rateLimitTimeout = Number(process.env.RATE_LIMIT_TIMEOUT) ?? 5000;
const stores = process.env.STORES ? process.env.STORES.split(',') : ['nvidia'];
const openBrowser = process.env.OPEN_BROWSER === 'true';
const isHeadless = process.env.HEADLESS ? process.env.HEADLESS === 'true' : true;
const showOnlyBrands = process.env.SHOW_ONLY_BRANDS ? process.env.SHOW_ONLY_BRANDS.split(',') : [];
const logLevel = process.env.LOG_LEVEL ?? 'info';
const store = {
showOnlyBrands: process.env.SHOW_ONLY_BRANDS ? process.env.SHOW_ONLY_BRANDS.split(',') : [],
stores: process.env.STORES ? process.env.STORES.split(',') : ['nvidia']
};
export const Config = {
isHeadless,
browser,
logLevel,
notifications,
openBrowser,
page,
rateLimitTimeout,
showOnlyBrands,
stores
store
};
+3 -3
View File
@@ -16,7 +16,7 @@ puppeteer.use(adblockerPlugin({blockTrackers: true}));
*/
async function main() {
const browser = await puppeteer.launch({
headless: Config.isHeadless,
headless: Config.browser.isHeadless,
defaultViewport: {
height: Config.page.height,
width: Config.page.width
@@ -26,7 +26,7 @@ async function main() {
const q = async.queue<Store>(async (store: Store, cb) => {
setTimeout(async () => {
try {
Logger.debug(`Scraping Initialized - ${store.name}`);
Logger.debug(`scraping initialized - ${store.name}`);
await lookup(browser, store);
} catch (error) {
// Ignoring errors; more than likely due to rate limits
@@ -35,7 +35,7 @@ async function main() {
cb();
q.push(store);
}
}, Config.rateLimitTimeout);
}, Config.browser.rateLimitTimeout);
}, Stores.length);
for (const store of Stores) {
+3 -3
View File
@@ -12,11 +12,11 @@ import {includesLabels} from './includes-labels';
* @param brand The brand of the GPU
*/
function filterBrand(brand: string) {
if (Config.showOnlyBrands.length === 0) {
if (Config.store.showOnlyBrands.length === 0) {
return true;
}
return Config.showOnlyBrands.includes(brand);
return Config.store.showOnlyBrands.includes(brand);
}
/**
@@ -70,7 +70,7 @@ export async function lookup(browser: Browser, store: Store) {
const givenUrl = link.cartUrl ? link.cartUrl : link.url;
if (Config.openBrowser) {
if (Config.browser.open) {
await open(givenUrl);
}
+1 -2
View File
@@ -1,4 +1,3 @@
import {Amazon} from './amazon';
import {AmazonCa} from './amazon-ca';
import {Asus} from './asus';
@@ -25,7 +24,7 @@ const masterList = new Map([
const list = new Map();
for (const name of Config.stores) {
for (const name of Config.store.stores) {
list.set(name, masterList.get(name));
}