mirror of
https://github.com/opelly27/streetmerchant.git
synced 2026-05-20 02:57:34 +00:00
fix: rateLimitTimeout not being defaulted (#106)
refactor: `browser` and `store` config object Signed-off-by: Jef LeCompte <jeffreylec@gmail.com>
This commit is contained in:
+14
-16
@@ -3,6 +3,14 @@ import {config} from 'dotenv';
|
||||
|
||||
config({path: resolve(__dirname, '../.env')});
|
||||
|
||||
const browser = {
|
||||
isHeadless: process.env.HEADLESS ? process.env.HEADLESS === 'true' : true,
|
||||
open: process.env.OPEN_BROWSER === 'true',
|
||||
rateLimitTimeout: process.env.RATE_LIMIT_TIMEOUT ? Number(process.env.RATE_LIMIT_TIMEOUT) : 5000
|
||||
};
|
||||
|
||||
const logLevel = process.env.LOG_LEVEL ?? 'info';
|
||||
|
||||
const notifications = {
|
||||
email: {
|
||||
username: process.env.EMAIL_USERNAME ?? '',
|
||||
@@ -45,25 +53,15 @@ const page = {
|
||||
userAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'
|
||||
};
|
||||
|
||||
const rateLimitTimeout = Number(process.env.RATE_LIMIT_TIMEOUT) ?? 5000;
|
||||
|
||||
const stores = process.env.STORES ? process.env.STORES.split(',') : ['nvidia'];
|
||||
|
||||
const openBrowser = process.env.OPEN_BROWSER === 'true';
|
||||
|
||||
const isHeadless = process.env.HEADLESS ? process.env.HEADLESS === 'true' : true;
|
||||
|
||||
const showOnlyBrands = process.env.SHOW_ONLY_BRANDS ? process.env.SHOW_ONLY_BRANDS.split(',') : [];
|
||||
|
||||
const logLevel = process.env.LOG_LEVEL ?? 'info';
|
||||
const store = {
|
||||
showOnlyBrands: process.env.SHOW_ONLY_BRANDS ? process.env.SHOW_ONLY_BRANDS.split(',') : [],
|
||||
stores: process.env.STORES ? process.env.STORES.split(',') : ['nvidia']
|
||||
};
|
||||
|
||||
export const Config = {
|
||||
isHeadless,
|
||||
browser,
|
||||
logLevel,
|
||||
notifications,
|
||||
openBrowser,
|
||||
page,
|
||||
rateLimitTimeout,
|
||||
showOnlyBrands,
|
||||
stores
|
||||
store
|
||||
};
|
||||
|
||||
+3
-3
@@ -16,7 +16,7 @@ puppeteer.use(adblockerPlugin({blockTrackers: true}));
|
||||
*/
|
||||
async function main() {
|
||||
const browser = await puppeteer.launch({
|
||||
headless: Config.isHeadless,
|
||||
headless: Config.browser.isHeadless,
|
||||
defaultViewport: {
|
||||
height: Config.page.height,
|
||||
width: Config.page.width
|
||||
@@ -26,7 +26,7 @@ async function main() {
|
||||
const q = async.queue<Store>(async (store: Store, cb) => {
|
||||
setTimeout(async () => {
|
||||
try {
|
||||
Logger.debug(`↗ Scraping Initialized - ${store.name}`);
|
||||
Logger.debug(`↗ scraping initialized - ${store.name}`);
|
||||
await lookup(browser, store);
|
||||
} catch (error) {
|
||||
// Ignoring errors; more than likely due to rate limits
|
||||
@@ -35,7 +35,7 @@ async function main() {
|
||||
cb();
|
||||
q.push(store);
|
||||
}
|
||||
}, Config.rateLimitTimeout);
|
||||
}, Config.browser.rateLimitTimeout);
|
||||
}, Stores.length);
|
||||
|
||||
for (const store of Stores) {
|
||||
|
||||
+3
-3
@@ -12,11 +12,11 @@ import {includesLabels} from './includes-labels';
|
||||
* @param brand The brand of the GPU
|
||||
*/
|
||||
function filterBrand(brand: string) {
|
||||
if (Config.showOnlyBrands.length === 0) {
|
||||
if (Config.store.showOnlyBrands.length === 0) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return Config.showOnlyBrands.includes(brand);
|
||||
return Config.store.showOnlyBrands.includes(brand);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -70,7 +70,7 @@ export async function lookup(browser: Browser, store: Store) {
|
||||
|
||||
const givenUrl = link.cartUrl ? link.cartUrl : link.url;
|
||||
|
||||
if (Config.openBrowser) {
|
||||
if (Config.browser.open) {
|
||||
await open(givenUrl);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
|
||||
import {Amazon} from './amazon';
|
||||
import {AmazonCa} from './amazon-ca';
|
||||
import {Asus} from './asus';
|
||||
@@ -25,7 +24,7 @@ const masterList = new Map([
|
||||
|
||||
const list = new Map();
|
||||
|
||||
for (const name of Config.stores) {
|
||||
for (const name of Config.store.stores) {
|
||||
list.set(name, masterList.get(name));
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user