opelly/streetmerchant
1
0
Fork 0
mirror of https://github.com/opelly27/streetmerchant.git synced 2026-05-20 16:57:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: refactor config and fix audits (#406)

Browse Source
This commit is contained in:
Jef LeCompte
2020-10-03 13:15:39 -04:00
committed by GitHub
parent f006a8595b
commit 8098a31092
28 changed files with 1765 additions and 403 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/store/fetch-links.ts
+6 -6
View File
@@ -1,5 +1,5 @@
import {Link, Series, Store} from './model';
import {Logger, Print} from '../logger';
import {Print, logger} from '../logger';
import {Browser} from 'puppeteer';
import cheerio from 'cheerio';
import {filterSeries} from './filter';
@@ -7,7 +7,7 @@ import {usingResponse} from '../util';
function addNewLinks(store: Store, links: Link[], series: Series) {
if (links.length === 0) {
Logger.error(Print.message('NO STORE LINKS FOUND', series, store, true));
logger.error(Print.message('NO STORE LINKS FOUND', series, store, true));
return;
}
@@ -19,8 +19,8 @@ function addNewLinks(store: Store, links: Link[], series: Series) {
return;
}
Logger.info(Print.message(`FOUND ${newLinks.length} STORE LINKS`, series, store, true));
Logger.debug(JSON.stringify(newLinks, null, 2));
logger.info(Print.message(`FOUND ${newLinks.length} STORE LINKS`, series, store, true));
logger.debug(JSON.stringify(newLinks, null, 2));
store.links = store.links.concat(newLinks);
}
@@ -37,13 +37,13 @@ export async function fetchLinks(store: Store, browser: Browser) {
continue;
}
Logger.info(Print.message('DETECTING STORE LINKS', series, store, true));
logger.info(Print.message('DETECTING STORE LINKS', series, store, true));
promises.push(usingResponse(browser, url, async response => {
const text = await response?.text();
if (!text) {
Logger.error(Print.message('NO RESPONSE', series, store, true));
logger.error(Print.message('NO RESPONSE', series, store, true));
return;
}
src/store/filter.ts
+7 -7
View File
@@ -1,5 +1,5 @@
import {Config} from '../config';
import {Link} from './model';
import {config} from '../config';
/**
* Returns true if the brand should be checked for stock
@@ -7,11 +7,11 @@ import {Link} from './model';
* @param brand The brand of the GPU
*/
function filterBrand(brand: Link['brand']): boolean {
if (Config.store.showOnlyBrands.length === 0) {
if (config.store.showOnlyBrands.length === 0) {
return true;
}
return Config.store.showOnlyBrands.includes(brand);
return config.store.showOnlyBrands.includes(brand);
}
/**
@@ -20,12 +20,12 @@ function filterBrand(brand: Link['brand']): boolean {
* @param model The model of the GPU
*/
function filterModel(model: Link['model']): boolean {
if (Config.store.showOnlyModels.length === 0) {
if (config.store.showOnlyModels.length === 0) {
return true;
}
const sanitizedModel = model.replace(/\s/g, '');
for (const configModel of Config.store.showOnlyModels) {
for (const configModel of config.store.showOnlyModels) {
const sanitizedConfigModel = configModel.replace(/\s/g, '');
if (sanitizedModel === sanitizedConfigModel) {
return true;
@@ -41,11 +41,11 @@ function filterModel(model: Link['model']): boolean {
* @param series The series of the GPU
*/
export function filterSeries(series: Link['series']): boolean {
if (Config.store.showOnlySeries.length === 0) {
if (config.store.showOnlySeries.length === 0) {
return true;
}
return Config.store.showOnlySeries.includes(series);
return config.store.showOnlySeries.includes(series);
}
/**
src/store/includes-labels.ts
+3 -3
View File
@@ -1,6 +1,6 @@
import {Element, LabelQuery, Pricing} from './model';
import {Logger} from '../logger';
import {Page} from 'puppeteer';
import {logger} from '../logger';
export type Selector = {
requireVisible: boolean;
@@ -44,7 +44,7 @@ export async function pageIncludesLabels(page: Page, query: LabelQuery, options:
return false;
}
Logger.debug(contents);
logger.debug(contents);
return includesLabels(contents, query.text);
}));
@@ -103,7 +103,7 @@ export async function cardPriceLimit(page: Page, query: Pricing, max: number, op
const priceSeperator = query.euroFormat ? /\./g : /,/g;
const cardpriceNumber = Number.parseFloat(cardPrice.replace(priceSeperator, '').match(/\d+/g)!.join('.'));
Logger.debug(`Raw card price: ${cardPrice} | Limit: ${max}`);
logger.debug(`Raw card price: ${cardPrice} | Limit: ${max}`);
return cardpriceNumber > max ? cardpriceNumber : null;
}
src/store/lookup.ts
+26 -26
View File
@@ -1,9 +1,9 @@
import {Browser, Page, Response} from 'puppeteer';
import {Link, Store} from './model';
import {Logger, Print} from '../logger';
import {Print, logger} from '../logger';
import {Selector, cardPriceLimit, pageIncludesLabels} from './includes-labels';
import {closePage, delay, getSleepTime, isStatusCodeInRange} from '../util';
import {Config} from '../config';
import {config} from '../config';
import {disableBlockerInPage} from '../adblocker';
import {filterStoreLink} from './filter';
import open from 'open';
@@ -27,20 +27,20 @@ async function lookup(browser: Browser, store: Store) {
continue;
}
if (Config.page.inStockWaitTime && inStock[link.url]) {
Logger.info(Print.inStockWaiting(link, store, true));
if (config.page.inStockWaitTime && inStock[link.url]) {
logger.info(Print.inStockWaiting(link, store, true));
continue;
}
const page = await browser.newPage();
page.setDefaultNavigationTimeout(Config.page.navigationTimeout);
await page.setUserAgent(Config.page.userAgent);
page.setDefaultNavigationTimeout(config.page.timeout);
await page.setUserAgent(config.page.userAgent);
if (store.disableAdBlocker) {
try {
await disableBlockerInPage(page);
} catch (error) {
Logger.error(error);
logger.error(error);
}
}
@@ -49,7 +49,7 @@ async function lookup(browser: Browser, store: Store) {
try {
statusCode = await lookupCard(browser, store, page, link);
} catch (error) {
Logger.error(`✖ [${store.name}] ${link.brand} ${link.model} - ${error.message as string}`);
logger.error(`✖ [${store.name}] ${link.brand} ${link.model} - ${error.message as string}`);
}
// Must apply backoff before closing the page, e.g. if CloudFlare is
@@ -67,16 +67,16 @@ async function lookupCard(browser: Browser, store: Store, page: Page, link: Link
const response: Response | null = await page.goto(link.url, {waitUntil: givenWaitFor});
if (!response) {
Logger.debug(Print.noResponse(link, store, true));
logger.debug(Print.noResponse(link, store, true));
}
const successStatusCodes = store.successStatusCodes ?? [[0, 399]];
const statusCode = response?.status() ?? 0;
if (!isStatusCodeInRange(statusCode, successStatusCodes)) {
if (statusCode === 429) {
Logger.warn(Print.rateLimit(link, store, true));
logger.warn(Print.rateLimit(link, store, true));
} else {
Logger.warn(Print.badStatusCode(link, store, statusCode, true));
logger.warn(Print.badStatusCode(link, store, statusCode, true));
}
return statusCode;
@@ -84,9 +84,9 @@ async function lookupCard(browser: Browser, store: Store, page: Page, link: Link
if (await lookupCardInStock(store, page, link)) {
const givenUrl = link.cartUrl ? link.cartUrl : link.url;
Logger.info(`${Print.inStock(link, store, true)}\n${givenUrl}`);
logger.info(`${Print.inStock(link, store, true)}\n${givenUrl}`);
if (Config.browser.open) {
if (config.browser.open) {
if (link.openCartAction === undefined) {
await open(givenUrl);
} else {
@@ -96,16 +96,16 @@ async function lookupCard(browser: Browser, store: Store, page: Page, link: Link
sendNotification(link, store);
if (Config.page.inStockWaitTime) {
if (config.page.inStockWaitTime) {
inStock[link.url] = true;
setTimeout(() => {
inStock[link.url] = false;
}, 1000 * Config.page.inStockWaitTime);
}, 1000 * config.page.inStockWaitTime);
}
if (Config.page.screenshot) {
Logger.debug('ℹ saving screenshot');
if (config.page.screenshot) {
logger.debug('ℹ saving screenshot');
link.screenshot = `success-${Date.now()}.png`;
await page.screenshot({path: link.screenshot});
@@ -126,36 +126,36 @@ async function lookupCardInStock(store: Store, page: Page, link: Link) {
const options = {...baseOptions, requireVisible: true, type: 'outerHTML' as const};
if (!await pageIncludesLabels(page, store.labels.inStock, options)) {
Logger.info(Print.outOfStock(link, store, true));
logger.info(Print.outOfStock(link, store, true));
return false;
}
}
if (store.labels.outOfStock) {
if (await pageIncludesLabels(page, store.labels.outOfStock, baseOptions)) {
Logger.info(Print.outOfStock(link, store, true));
logger.info(Print.outOfStock(link, store, true));
return false;
}
}
if (store.labels.bannedSeller) {
if (await pageIncludesLabels(page, store.labels.bannedSeller, baseOptions)) {
Logger.warn(Print.bannedSeller(link, store, true));
logger.warn(Print.bannedSeller(link, store, true));
return false;
}
}
if (store.labels.maxPrice) {
const priceLimit = await cardPriceLimit(page, store.labels.maxPrice, Config.store.maxPrice, baseOptions);
const priceLimit = await cardPriceLimit(page, store.labels.maxPrice, config.store.maxPrice, baseOptions);
if (priceLimit) {
Logger.info(Print.maxPrice(link, store, priceLimit, true));
logger.info(Print.maxPrice(link, store, priceLimit, true));
return false;
}
}
if (store.labels.captcha) {
if (await pageIncludesLabels(page, store.labels.captcha, baseOptions)) {
Logger.warn(Print.captcha(link, store, true));
logger.warn(Print.captcha(link, store, true));
await delay(getSleepTime());
return false;
}
@@ -165,14 +165,14 @@ async function lookupCardInStock(store: Store, page: Page, link: Link) {
}
export async function tryLookupAndLoop(browser: Browser, store: Store) {
Logger.debug(`[${store.name}] Starting lookup...`);
logger.debug(`[${store.name}] Starting lookup...`);
try {
await lookup(browser, store);
} catch (error) {
Logger.error(error);
logger.error(error);
}
const sleepTime = getSleepTime();
Logger.debug(`[${store.name}] Lookup done, next one in ${sleepTime} ms`);
logger.debug(`[${store.name}] Lookup done, next one in ${sleepTime} ms`);
setTimeout(tryLookupAndLoop, sleepTime, browser, store);
}
src/store/model/helpers/backoff.ts
+6 -6
View File
@@ -1,7 +1,7 @@
import {Link, Store} from '..';
import {Logger, Print} from '../../../logger';
import {Print, logger} from '../../../logger';
import {delay, isStatusCodeInRange} from '../../../util';
import {Config} from '../../../config';
import {config} from '../../../config';
type Backoff = {
count: number;
@@ -27,26 +27,26 @@ export async function processBackoffDelay(store: Store, link: Link, statusCode:
let backoff = stores[store.name];
if (!backoff) {
backoff = {count: 0, time: Config.browser.minBackoff};
backoff = {count: 0, time: config.browser.minBackoff};
stores[store.name] = backoff;
}
if (!isBackoff) {
if (backoff.count > 0) {
backoff.count--;
backoff.time = Math.max(backoff.time / 2, Config.browser.minBackoff);
backoff.time = Math.max(backoff.time / 2, config.browser.minBackoff);
}
return -1;
}
const backoffTime = backoff.time;
Logger.debug(Print.backoff(link, store, {delay: backoffTime, statusCode}, true));
logger.debug(Print.backoff(link, store, {delay: backoffTime, statusCode}, true));
await delay(backoff.time);
backoff.count++;
backoff.time = Math.min(backoff.time * 2, Config.browser.maxBackoff);
backoff.time = Math.min(backoff.time * 2, config.browser.maxBackoff);
return backoffTime;
}
src/store/model/helpers/nvidia-cart.ts
+17 -17
View File
@@ -1,8 +1,8 @@
import {NvidiaRegionInfo, regionInfos} from '../nvidia-api';
import {usingPage, usingResponse} from '../../../util';
import {Browser} from 'puppeteer';
import {Config} from '../../../config';
import {Logger} from '../../../logger';
import {config} from '../../../config';
import {logger} from '../../../logger';
import open from 'open';
interface NvidiaSessionTokenJSON {
@@ -34,7 +34,7 @@ export class NvidiaCart {
await this.refreshSessionToken();
setTimeout(callback, Config.nvidia.sessionTtl);
setTimeout(callback, config.nvidia.sessionTtl);
};
this.isKeepAlive = true;
@@ -47,7 +47,7 @@ export class NvidiaCart {
}
public get regionInfo(): NvidiaRegionInfo {
const country = Config.store.country;
const country = config.store.country;
const regionInfo = regionInfos.get(country);
if (!regionInfo) {
throw new Error(`Unknown country ${country}`);
@@ -62,20 +62,20 @@ export class NvidiaCart {
public async addToCard(productId: number, name: string): Promise<string> {
let cartUrl: string | undefined;
Logger.info(`🚀🚀🚀 [nvidia] ${name}, starting auto add to cart 🚀🚀🚀`);
logger.info(`🚀🚀🚀 [nvidia] ${name}, starting auto add to cart 🚀🚀🚀`);
try {
Logger.info(`🚀🚀🚀 [nvidia] ${name}, adding to cart 🚀🚀🚀`);
logger.info(`🚀🚀🚀 [nvidia] ${name}, adding to cart 🚀🚀🚀`);
let lastError: Error | string | undefined;
/* eslint-disable no-await-in-loop */
for (let i = 0; i < Config.nvidia.addToCardAttempts; i++) {
for (let i = 0; i < config.nvidia.addToCardAttempts; i++) {
try {
cartUrl = await this.addToCartAndGetLocationRedirect(productId);
break;
} catch (error) {
Logger.error(`✖ [nvidia] ${name} could not automatically add to cart, attempt ${i + 1} of ${Config.nvidia.addToCardAttempts}`, error);
Logger.debug(error);
logger.error(`✖ [nvidia] ${name} could not automatically add to cart, attempt ${i + 1} of ${config.nvidia.addToCardAttempts}`, error);
logger.debug(error);
lastError = error;
}
@@ -87,13 +87,13 @@ export class NvidiaCart {
throw lastError;
}
Logger.info(`🚀🚀🚀 [nvidia] ${name}, opening checkout page 🚀🚀🚀`);
Logger.info(cartUrl);
logger.info(`🚀🚀🚀 [nvidia] ${name}, opening checkout page 🚀🚀🚀`);
logger.info(cartUrl);
await open(cartUrl);
} catch (error) {
Logger.error(`✖ [nvidia] ${name} could not automatically add to cart, opening page`);
Logger.debug(error);
logger.error(`✖ [nvidia] ${name} could not automatically add to cart, opening page`);
logger.debug(error);
cartUrl = this.fallbackCartUrl;
@@ -116,7 +116,7 @@ export class NvidiaCart {
}
public async refreshSessionToken(): Promise<void> {
Logger.debug('ℹ [nvidia] refreshing session token');
logger.debug('ℹ [nvidia] refreshing session token');
try {
const result = await usingResponse(this.browser, this.sessionUrl, async response => {
return response?.json() as NvidiaSessionTokenJSON | undefined;
@@ -126,10 +126,10 @@ export class NvidiaCart {
}
this.sessionToken = result.session_token;
Logger.debug(`ℹ [nvidia] session_token=${result.session_token}`);
logger.debug(`ℹ [nvidia] session_token=${result.session_token}`);
} catch (error) {
const message: string = typeof error === 'object' ? error.message : error;
Logger.error(`✖ [nvidia] ${message}`);
logger.error(`✖ [nvidia] ${message}`);
}
}
@@ -137,7 +137,7 @@ export class NvidiaCart {
const url = 'https://api-prod.nvidia.com/direct-sales-shop/DR/add-to-cart';
const sessionToken = await this.getSessionToken();
Logger.info(`ℹ [nvidia] session_token=${sessionToken}`);
logger.info(`ℹ [nvidia] session_token=${sessionToken}`);
const locationData = await usingPage(this.browser, async page => {
page.removeAllListeners('request');
src/store/model/helpers/nvidia.ts
+3 -3
View File
@@ -1,12 +1,12 @@
import {NvidiaRegionInfo, regionInfos} from '../nvidia-api';
import {Browser} from 'puppeteer';
import {Config} from '../../../config';
import {Link} from '../store';
import {NvidiaCart} from './nvidia-cart';
import {config} from '../../../config';
import {timestampUrlParameter} from '../../timestamp-url-parameter';
function getRegionInfo(): NvidiaRegionInfo {
let country = Config.store.country;
let country = config.store.country;
if (!regionInfos.has(country)) {
country = 'usa';
}
@@ -30,7 +30,7 @@ export function generateSetupAction() {
return async (browser: Browser) => {
cart = new NvidiaCart(browser);
if (Config.browser.open) {
if (config.browser.open) {
cart.keepAlive();
}
};
src/store/model/index.ts
+11 -11
View File
@@ -8,11 +8,9 @@ import {AsusDe} from './asus-de';
import {BAndH} from './bandh';
import {BestBuy} from './bestbuy';
import {BestBuyCa} from './bestbuy-ca';
import {Config} from '../../config';
import {Evga} from './evga';
import {EvgaEu} from './evga-eu';
import {Gamestop} from './gamestop';
import {Logger} from '../../logger';
import {MicroCenter} from './microcenter';
import {Newegg} from './newegg';
import {NeweggCa} from './newegg-ca';
@@ -22,6 +20,8 @@ import {OfficeDepot} from './officedepot';
import {Pny} from './pny';
import {Store} from './store';
import {Zotac} from './zotac';
import {config} from '../../config';
import {logger} from '../../logger';
const masterList = new Map([
[Adorama.name, Adorama],
@@ -49,27 +49,27 @@ const masterList = new Map([
const list = new Map();
for (const name of Config.store.stores) {
for (const name of config.store.stores) {
if (masterList.has(name)) {
list.set(name, masterList.get(name));
} else {
const logString = `No store named ${name}, skipping.`;
Logger.warn(logString);
logger.warn(logString);
}
}
Logger.info(`ℹ selected stores: ${Array.from(list.keys()).join(', ')}`);
logger.info(`ℹ selected stores: ${Array.from(list.keys()).join(', ')}`);
if (Config.store.showOnlyBrands.length > 0) {
Logger.info(`ℹ selected brands: ${Config.store.showOnlyBrands.join(', ')}`);
if (config.store.showOnlyBrands.length > 0) {
logger.info(`ℹ selected brands: ${config.store.showOnlyBrands.join(', ')}`);
}
if (Config.store.showOnlyModels.length > 0) {
Logger.info(`ℹ selected models: ${Config.store.showOnlyModels.join(', ')}`);
if (config.store.showOnlyModels.length > 0) {
logger.info(`ℹ selected models: ${config.store.showOnlyModels.join(', ')}`);
}
if (Config.store.showOnlySeries.length > 0) {
Logger.info(`ℹ selected series: ${Config.store.showOnlySeries.join(', ')}`);
if (config.store.showOnlySeries.length > 0) {
logger.info(`ℹ selected series: ${config.store.showOnlySeries.join(', ')}`);
}
export const Stores = Array.from(list.values()) as Store[];
src/store/model/microcenter.ts
+5 -4
View File
@@ -1,7 +1,8 @@
import {Config} from '../../config';
import {Store} from './store';
import {config} from '../../config';
const microCenterLocation = config.store.microCenterLocation;
const MicroCenterLocation = Config.store.microCenterLocation;
const microCenterLocationToId: Map<string, string> = new Map([
['web', '029'],
['brooklyn', '115'],
@@ -32,10 +33,10 @@ const microCenterLocationToId: Map<string, string> = new Map([
]);
let storeId: string;
if (microCenterLocationToId.get(MicroCenterLocation) === undefined) {
if (microCenterLocationToId.get(microCenterLocation) === undefined) {
storeId = '029';
} else {
storeId = microCenterLocationToId.get(MicroCenterLocation)!;
storeId = microCenterLocationToId.get(microCenterLocation)!;
}
export const MicroCenter: Store = {