opelly/streetmerchant
1
0
Fork 0
mirror of https://github.com/opelly27/streetmerchant.git synced 2026-05-20 01:47:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: refactor config and fix audits (#406)

Browse Source
This commit is contained in:
Jef LeCompte
2020-10-03 13:15:39 -04:00
committed by GitHub
parent f006a8595b
commit 8098a31092
28 changed files with 1765 additions and 403 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package-lock.json
Generated
+1392 -30
View File
File diff suppressed because it is too large Load Diff
package.json
+2 -1
View File
@@ -23,6 +23,7 @@
},
"homepage": "https://github.com/jef/nvidia-snatcher#readme",
"dependencies": {
"@hijef/pushbullet": "^2.4.2",
"@slack/web-api": "^5.12.0",
"chalk": "^4.1.0",
"cheerio": "^1.0.0-rc.3",
@@ -38,7 +39,6 @@
"puppeteer-extra-plugin-adblocker": "^2.11.6",
"puppeteer-extra-plugin-block-resources": "^2.2.7",
"puppeteer-extra-plugin-stealth": "^2.6.2",
"pushbullet": "^2.4.0",
"pushover-notifications": "^1.2.2",
"twilio": "^3.49.4",
"twitter": "^1.7.1",
@@ -57,6 +57,7 @@
"rimraf": "^3.0.2",
"ts-node": "^9.0.0",
"typescript": "^4.0.2",
"webpack": "^4.44.2",
"xo": "^0.33.1"
},
"xo": {
src/config.ts
+5 -5
View File
@@ -1,10 +1,10 @@
import {banner} from './banner';
console.log(banner);
import {config} from 'dotenv';
import {config as config_} from 'dotenv';
import path from 'path';
config({path: path.resolve(__dirname, '../.env')});
config_({path: path.resolve(__dirname, '../.env')});
/**
* Returns environment variable, given array, or default array.
@@ -91,7 +91,7 @@ const notifications = {
number: envOrString(process.env.PHONE_NUMBER)
},
playSound: envOrString(process.env.PLAY_SOUND),
pushBulletApiKey: envOrString(process.env.PUSHBULLET),
pushbullet: envOrString(process.env.PUSHBULLET),
pushover: {
priority: envOrString(process.env.PUSHOVER_PRIORITY),
token: envOrString(process.env.PUSHOVER_TOKEN),
@@ -128,8 +128,8 @@ const nvidia = {
const page = {
height: 1080,
inStockWaitTime: envOrNumber(process.env.IN_STOCK_WAIT_TIME),
navigationTimeout: envOrNumber(process.env.PAGE_TIMEOUT, 30000),
screenshot: envOrBoolean(process.env.SCREENSHOT),
timeout: envOrNumber(process.env.PAGE_TIMEOUT, 30000),
userAgent: envOrString(process.env.USER_AGENT, 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'),
width: 1920
};
@@ -149,7 +149,7 @@ const store = {
stores: envOrArray(process.env.STORES, ['nvidia'])
};
export const Config = {
export const config = {
browser,
logLevel,
notifications,
src/index.ts
+12 -12
View File
@@ -1,16 +1,16 @@
import {Config} from './config';
import {Logger} from './logger';
import {Stores} from './store/model';
import {adBlocker} from './adblocker';
import {config} from './config';
import {fetchLinks} from './store/fetch-links';
import {getSleepTime} from './util';
import {logger} from './logger';
import puppeteer from 'puppeteer-extra';
import resourceBlock from 'puppeteer-extra-plugin-block-resources';
import stealthPlugin from 'puppeteer-extra-plugin-stealth';
import {tryLookupAndLoop} from './store';
puppeteer.use(stealthPlugin());
if (Config.browser.lowBandwidth) {
if (config.browser.lowBandwidth) {
puppeteer.use(resourceBlock({
blockedTypes: new Set(['image', 'font'])
}));
@@ -23,7 +23,7 @@ if (Config.browser.lowBandwidth) {
*/
async function main() {
if (Stores.length === 0) {
Logger.error('✖ no stores selected', Stores);
logger.error('✖ no stores selected', Stores);
return;
}
@@ -31,28 +31,28 @@ async function main() {
// Skip Chromium Linux Sandbox
// https://github.com/puppeteer/puppeteer/blob/main/docs/troubleshooting.md#setting-up-chrome-linux-sandbox
if (Config.browser.isTrusted) {
if (config.browser.isTrusted) {
args.push('--no-sandbox');
args.push('--disable-setuid-sandbox');
}
// Add the address of the proxy server if defined
if (Config.proxy.address) {
args.push(`--proxy-server=http://${Config.proxy.address}:${Config.proxy.port}`);
if (config.proxy.address) {
args.push(`--proxy-server=http://${config.proxy.address}:${config.proxy.port}`);
}
const browser = await puppeteer.launch({
args,
defaultViewport: {
height: Config.page.height,
width: Config.page.width
height: config.page.height,
width: config.page.width
},
headless: Config.browser.isHeadless
headless: config.browser.isHeadless
});
const promises = [];
for (const store of Stores) {
Logger.debug(store.links);
logger.debug(store.links);
if (store.setupAction !== undefined) {
store.setupAction(browser);
}
@@ -73,6 +73,6 @@ async function main() {
try {
void main();
} catch (error) {
Logger.error('✖ something bad happened, resetting nvidia-snatcher', error);
logger.error('✖ something bad happened, resetting nvidia-snatcher', error);
void main();
}
src/logger.ts
+5 -5
View File
@@ -1,7 +1,7 @@
import {Link, Store} from './store/model';
import winston, {format} from 'winston';
import {Config} from './config';
import chalk from 'chalk';
import {config} from './config';
const prettyJson = format.printf(info => {
const timestamp = new Date().toLocaleTimeString();
@@ -13,7 +13,7 @@ const prettyJson = format.printf(info => {
return chalk.grey(`[${timestamp}]`) + ` ${info.level} ` + chalk.grey('::') + ` ${info.message}`;
});
export const Logger = winston.createLogger({
export const logger = winston.createLogger({
format: format.combine(
format.colorize(),
format.prettyPrint(),
@@ -21,7 +21,7 @@ export const Logger = winston.createLogger({
format.simple(),
prettyJson
),
level: Config.logLevel,
level: config.logLevel,
transports: [new winston.transports.Console({})]
});
@@ -76,10 +76,10 @@ export const Print = {
},
maxPrice(link: Link, store: Store, price: number, color?: boolean): string {
if (color) {
return '✖ ' + buildProductString(link, store, true) + ' :: ' + chalk.yellow(`PRICE ${price} EXCEEDS LIMIT ${Config.store.maxPrice}`);
return '✖ ' + buildProductString(link, store, true) + ' :: ' + chalk.yellow(`PRICE ${price} EXCEEDS LIMIT ${config.store.maxPrice}`);
}
return `✖ ${buildProductString(link, store)} :: PRICE ${price} EXCEEDS LIMIT ${Config.store.maxPrice}`;
return `✖ ${buildProductString(link, store)} :: PRICE ${price} EXCEEDS LIMIT ${config.store.maxPrice}`;
},
message(message: string, topic: string, store: Store, color?: boolean): string {
if (color) {
src/notification/desktop.ts
+15 -9
View File
@@ -1,14 +1,20 @@
import {Link, Store} from '../store/model';
import {Logger, Print} from '../logger';
import {Print, logger} from '../logger';
import {config} from '../config';
import notifier from 'node-notifier';
export function sendDesktopNotification(link: Link, store: Store) {
(async () => {
notifier.notify({
message: link.cartUrl ? link.cartUrl : link.url,
title: Print.inStock(link, store)
});
const desktop = config.notifications.desktop;
Logger.info('✔ desktop notification sent');
})();
export function sendDesktopNotification(link: Link, store: Store) {
if (desktop) {
logger.debug('↗ sending desktop notification');
(async () => {
notifier.notify({
message: link.cartUrl ? link.cartUrl : link.url,
title: Print.inStock(link, store)
});
logger.info('✔ desktop notification sent');
})();
}
}
src/notification/discord.ts
+35 -30
View File
@@ -1,38 +1,43 @@
import {Link, Store} from '../store/model';
import {MessageBuilder, Webhook} from 'discord-webhook-node';
import {Config} from '../config';
import {Logger} from '../logger';
import {config} from '../config';
import {logger} from '../logger';
const hooks = Config.notifications.discord.webHookUrl;
const notifyGroup = Config.notifications.discord.notifyGroup;
const discord = config.notifications.discord;
const hooks = discord.webHookUrl;
const notifyGroup = discord.notifyGroup;
export function sendDiscordMessage(link: Link, store: Store) {
(async () => {
try {
const embed = new MessageBuilder();
embed.setTitle('Stock Notification');
embed.addField('URL', link.cartUrl ? link.cartUrl : link.url, true);
embed.addField('Store', store.name, true);
embed.addField('Brand', link.brand, true);
embed.addField('Model', link.model, true);
if (discord.webHookUrl.length > 0) {
logger.debug('↗ sending discord message');
if (notifyGroup) {
embed.setText(notifyGroup.join(' '));
(async () => {
try {
const embed = new MessageBuilder();
embed.setTitle('Stock Notification');
embed.addField('URL', link.cartUrl ? link.cartUrl : link.url, true);
embed.addField('Store', store.name, true);
embed.addField('Brand', link.brand, true);
embed.addField('Model', link.model, true);
if (notifyGroup) {
embed.setText(notifyGroup.join(' '));
}
embed.setColor(0x76B900);
embed.setTimestamp();
const promises = [];
for (const hook of hooks) {
promises.push(new Webhook(hook).send(embed));
}
await Promise.all(promises);
logger.info('✔ discord message sent');
} catch (error) {
logger.error('✖ couldn\'t send discord message', error);
}
embed.setColor(0x76B900);
embed.setTimestamp();
const promises = [];
for (const hook of hooks) {
promises.push(new Webhook(hook).send(embed));
}
await Promise.all(promises);
Logger.info('✔ discord message sent');
} catch (error) {
Logger.error('✖ couldn\'t send discord message', error);
}
})();
})();
}
}
src/notification/email.ts
+26 -22
View File
@@ -1,10 +1,10 @@
import {Link, Store} from '../store/model';
import {Logger, Print} from '../logger';
import {Config} from '../config';
import {Print, logger} from '../logger';
import Mail from 'nodemailer/lib/mailer';
import {config} from '../config';
import nodemailer from 'nodemailer';
const email = Config.notifications.email;
const email = config.notifications.email;
const transporter = nodemailer.createTransport({
auth: {
@@ -15,24 +15,28 @@ const transporter = nodemailer.createTransport({
});
export function sendEmail(link: Link, store: Store) {
const mailOptions: Mail.Options = {
attachments: link.screenshot ? [
{
filename: link.screenshot,
path: `./${link.screenshot}`
}
] : undefined,
from: email.username,
subject: Print.inStock(link, store),
text: link.cartUrl ? link.cartUrl : link.url,
to: email.to
};
if (email.username && email.password) {
logger.debug('↗ sending email');
transporter.sendMail(mailOptions, error => {
if (error) {
Logger.error('✖ couldn\'t send email', error);
} else {
Logger.info('✔ email sent');
}
});
const mailOptions: Mail.Options = {
attachments: link.screenshot ? [
{
filename: link.screenshot,
path: `./${link.screenshot}`
}
] : undefined,
from: email.username,
subject: Print.inStock(link, store),
text: link.cartUrl ? link.cartUrl : link.url,
to: email.to
};
transporter.sendMail(mailOptions, error => {
if (error) {
logger.error('✖ couldn\'t send email', error);
} else {
logger.info('✔ email sent');
}
});
}
}
src/notification/notification.ts
+12 -67
View File
@@ -1,11 +1,9 @@
import {Link, Store} from '../store/model';
import {Config} from '../config';
import {Logger} from '../logger';
import {playSound} from './sound';
import {sendDesktopNotification} from './desktop';
import {sendDiscordMessage} from './discord';
import {sendEmail} from './email';
import {sendPushBulletNotification} from './pushbullet';
import {sendPushbulletNotification} from './pushbullet';
import {sendPushoverNotification} from './pushover';
import {sendSMS} from './sms';
import {sendSlackMessage} from './slack';
@@ -13,69 +11,16 @@ import {sendTelegramMessage} from './telegram';
import {sendTweet} from './twitter';
import {sendTwilioMessage} from './twilio';
const notifications = Config.notifications;
export function sendNotification(link: Link, store: Store) {
if (notifications.email.username && notifications.email.password) {
Logger.debug('↗ sending email');
sendEmail(link, store);
}
if (notifications.phone.number) {
Logger.debug('↗ sending sms');
const carrier = notifications.phone.carrier;
if (carrier && notifications.phone.availableCarriers.has(carrier)) {
sendSMS(link, store);
}
}
if (notifications.playSound) {
Logger.debug('↗ playing sound');
playSound();
}
if (notifications.desktop) {
Logger.debug('↗ sending desktop notification');
sendDesktopNotification(link, store);
}
if (notifications.discord.webHookUrl.length > 0) {
Logger.debug('↗ sending discord message');
sendDiscordMessage(link, store);
}
if (notifications.slack.channel && notifications.slack.token) {
Logger.debug('↗ sending slack message');
sendSlackMessage(link, store);
}
if (notifications.telegram.accessToken && notifications.telegram.chatId) {
Logger.debug('↗ sending telegram message');
sendTelegramMessage(link, store);
}
if (notifications.twilio.accountSid && notifications.twilio.authToken) {
Logger.debug('↗ sending twilio message');
sendTwilioMessage(link, store);
}
if (notifications.pushBulletApiKey) {
Logger.debug('↗ sending pushbullet message');
sendPushBulletNotification(link, store);
}
if (notifications.pushover.token && notifications.pushover.username) {
Logger.debug('↗ sending pushover message');
sendPushoverNotification(link, store);
}
if (
notifications.twitter.accessTokenKey &&
notifications.twitter.accessTokenSecret &&
notifications.twitter.consumerKey &&
notifications.twitter.consumerSecret
) {
Logger.debug('↗ sending twitter message');
sendTweet(link, store);
}
sendEmail(link, store);
sendSMS(link, store);
playSound();
sendDesktopNotification(link, store);
sendDiscordMessage(link, store);
sendSlackMessage(link, store);
sendTelegramMessage(link, store);
sendTwilioMessage(link, store);
sendPushbulletNotification(link, store);
sendPushoverNotification(link, store);
sendTweet(link, store);
}
src/notification/pushbullet.ts
+21 -17
View File
@@ -1,22 +1,26 @@
import {Link, Store} from '../store/model';
import {Logger, Print} from '../logger';
import {Config} from '../config';
import PushBullet from 'pushbullet';
import {Print, logger} from '../logger';
import PushBullet from '@hijef/pushbullet';
import {config} from '../config';
const pushBulletApiKey = Config.notifications.pushBulletApiKey;
const pushbullet = config.notifications.pushbullet;
export function sendPushBulletNotification(link: Link, store: Store) {
const pusher = new PushBullet(pushBulletApiKey);
export function sendPushbulletNotification(link: Link, store: Store) {
if (pushbullet) {
logger.debug('↗ sending pushbullet message');
pusher.note(
{},
Print.inStock(link, store),
link.cartUrl ? link.cartUrl : link.url,
(error: Error) => {
if (error) {
Logger.error('✖ couldn\'t send pushbullet message', error);
} else {
Logger.info('✔ pushbullet message sent');
}
});
const pusher = new PushBullet(pushbullet);
pusher.note(
{},
Print.inStock(link, store),
link.cartUrl ? link.cartUrl : link.url,
(error: Error) => {
if (error) {
logger.error('✖ couldn\'t send pushbullet message', error);
} else {
logger.info('✔ pushbullet message sent');
}
});
}
}
src/notification/pushover.ts
+19 -15
View File
@@ -1,26 +1,30 @@
import {Link, Store} from '../store/model';
import {Logger, Print} from '../logger';
import {Config} from '../config';
import {Print, logger} from '../logger';
import Push from 'pushover-notifications';
import {config} from '../config';
const pushover = Config.notifications.pushover;
const pushover = config.notifications.pushover;
const push = new Push({
token: pushover.token,
user: pushover.username
});
export function sendPushoverNotification(link: Link, store: Store) {
const message = {
message: link.cartUrl ? link.cartUrl : link.url,
priority: pushover.priority,
title: Print.inStock(link, store)
};
if (pushover.token && pushover.username) {
logger.debug('↗ sending pushover message');
push.send(message, (error: Error) => {
if (error) {
Logger.error('✖ couldn\'t send pushover message', error);
} else {
Logger.info('✔ pushover message sent');
}
});
const message = {
message: link.cartUrl ? link.cartUrl : link.url,
priority: pushover.priority,
title: Print.inStock(link, store)
};
push.send(message, (error: Error) => {
if (error) {
logger.error('✖ couldn\'t send pushover message', error);
} else {
logger.info('✔ pushover message sent');
}
});
}
}
src/notification/slack.ts
+25 -20
View File
@@ -1,30 +1,35 @@
import {Link, Store} from '../store/model';
import {Logger, Print} from '../logger';
import {Config} from '../config';
import {Print, logger} from '../logger';
import {WebClient} from '@slack/web-api';
import {config} from '../config';
const channel = Config.notifications.slack.channel;
const token = Config.notifications.slack.token;
const slack = config.notifications.slack;
const channel = slack.channel;
const token = slack.token;
const web = new WebClient(token);
export function sendSlackMessage(link: Link, store: Store) {
(async () => {
const givenUrl = link.cartUrl ? link.cartUrl : link.url;
if (slack.channel && slack.token) {
logger.debug('↗ sending slack message');
try {
const result = await web.chat.postMessage({
channel,
text: `${Print.inStock(link, store)}\n${givenUrl}`
});
(async () => {
const givenUrl = link.cartUrl ? link.cartUrl : link.url;
if (!result.ok) {
Logger.error('✖ couldn\'t send slack message', result);
return;
try {
const result = await web.chat.postMessage({
channel,
text: `${Print.inStock(link, store)}\n${givenUrl}`
});
if (!result.ok) {
logger.error('✖ couldn\'t send slack message', result);
return;
}
logger.info('✔ slack message sent');
} catch (error) {
logger.error('✖ couldn\'t send slack message', error);
}
Logger.info('✔ slack message sent');
} catch (error) {
Logger.error('✖ couldn\'t send slack message', error);
}
})();
})();
}
}
src/notification/sms.ts
+31 -24
View File
@@ -1,14 +1,14 @@
import {Link, Store} from '../store/model';
import {Logger, Print} from '../logger';
import {Config} from '../config';
import {Print, logger} from '../logger';
import Mail from 'nodemailer/lib/mailer';
import {config} from '../config';
import nodemailer from 'nodemailer';
if (Config.notifications.phone.number && !Config.notifications.email.username) {
Logger.warn('✖ in order to recieve sms alerts, email notifications must also be configured');
if (config.notifications.phone.number && !config.notifications.email.username) {
logger.warn('✖ in order to recieve sms alerts, email notifications must also be configured');
}
const [email, phone] = [Config.notifications.email, Config.notifications.phone];
const [email, phone] = [config.notifications.email, config.notifications.phone];
const transporter = nodemailer.createTransport({
auth: {
@@ -19,26 +19,33 @@ const transporter = nodemailer.createTransport({
});
export function sendSMS(link: Link, store: Store) {
const mailOptions: Mail.Options = {
attachments: link.screenshot ? [
{
filename: link.screenshot,
path: `./${link.screenshot}`
}
] : undefined,
from: email.username,
subject: Print.inStock(link, store, false, true),
text: link.cartUrl ? link.cartUrl : link.url,
to: generateAddress()
};
if (phone.number) {
logger.debug('↗ sending sms');
const carrier = phone.carrier;
transporter.sendMail(mailOptions, error => {
if (error) {
Logger.error('✖ couldn\'t send sms', error);
} else {
Logger.info('✔ sms sent');
if (carrier && phone.availableCarriers.has(carrier)) {
const mailOptions: Mail.Options = {
attachments: link.screenshot ? [
{
filename: link.screenshot,
path: `./${link.screenshot}`
}
] : undefined,
from: email.username,
subject: Print.inStock(link, store, false, true),
text: link.cartUrl ? link.cartUrl : link.url,
to: generateAddress()
};
transporter.sendMail(mailOptions, error => {
if (error) {
logger.error('✖ couldn\'t send sms', error);
} else {
logger.info('✔ sms sent');
}
});
}
});
}
}
function generateAddress() {
@@ -48,5 +55,5 @@ function generateAddress() {
return [phone.number, phone.availableCarriers.get(carrier)].join('@');
}
Logger.error('✖ unknown carrier', carrier);
logger.error('✖ unknown carrier', carrier);
}
src/notification/sound.ts
+13 -11
View File
@@ -1,35 +1,37 @@
import {Config} from '../config';
import {Logger} from '../logger';
import {config} from '../config';
import fs from 'fs';
import {logger} from '../logger';
import playerLib from 'play-sound';
let player: any;
if (Config.notifications.playSound) {
if (config.notifications.playSound) {
player = playerLib();
if (player.player === null) {
Logger.warn('✖ couldn\'t find sound player');
logger.warn('✖ couldn\'t find sound player');
} else {
const playerName: string = player.player;
Logger.info(`✔ sound player found: ${playerName}`);
logger.info(`✔ sound player found: ${playerName}`);
}
}
export function playSound() {
if (player.player !== null) {
fs.access(Config.notifications.playSound, fs.constants.F_OK, error => {
if (config.notifications.playSound && player.player !== null) {
logger.debug('↗ playing sound');
fs.access(config.notifications.playSound, fs.constants.F_OK, error => {
if (error) {
Logger.error(`✖ error opening sound file: ${error.message}`);
logger.error(`✖ error opening sound file: ${error.message}`);
return;
}
player.play(Config.notifications.playSound, (error: Error) => {
player.play(config.notifications.playSound, (error: Error) => {
if (error) {
Logger.error('✖ couldn\'t play sound', error);
logger.error('✖ couldn\'t play sound', error);
}
Logger.info('✔ played sound');
logger.info('✔ played sound');
});
});
}
src/notification/telegram.ts
+16 -12
View File
@@ -1,23 +1,27 @@
import {Link, Store} from '../store/model';
import {Logger, Print} from '../logger';
import {Config} from '../config';
import {Print, logger} from '../logger';
import {TelegramClient} from 'messaging-api-telegram';
import {config} from '../config';
const telegram = Config.notifications.telegram;
const telegram = config.notifications.telegram;
const client = new TelegramClient({
accessToken: telegram.accessToken
});
export function sendTelegramMessage(link: Link, store: Store) {
(async () => {
const givenUrl = link.cartUrl ? link.cartUrl : link.url;
if (telegram.accessToken && telegram.chatId) {
logger.debug('↗ sending telegram message');
try {
await client.sendMessage(telegram.chatId, `${Print.inStock(link, store)}\n${givenUrl}`);
Logger.info('✔ telegram message sent');
} catch (error) {
Logger.error('✖ couldn\'t send telegram message', error);
}
})();
(async () => {
const givenUrl = link.cartUrl ? link.cartUrl : link.url;
try {
await client.sendMessage(telegram.chatId, `${Print.inStock(link, store)}\n${givenUrl}`);
logger.info('✔ telegram message sent');
} catch (error) {
logger.error('✖ couldn\'t send telegram message', error);
}
})();
}
}
src/notification/twilio.ts
+27 -19
View File
@@ -1,25 +1,33 @@
import {Link, Store} from '../store/model';
import {Logger, Print} from '../logger';
import {Config} from '../config';
import twilio from 'twilio';
import {Print, logger} from '../logger';
import {Twilio} from 'twilio';
import {config} from '../config';
const config = Config.notifications.twilio;
const twilio = config.notifications.twilio;
let client: Twilio;
if (twilio.accountSid && twilio.authToken) {
client = new Twilio(twilio.accountSid, twilio.authToken);
}
export function sendTwilioMessage(link: Link, store: Store) {
(async () => {
const givenUrl = link.cartUrl ? link.cartUrl : link.url;
const message = `${Print.inStock(link, store)}\n${givenUrl}`;
if (client) {
logger.debug('↗ sending twilio message');
try {
const client = twilio(config.accountSid, config.authToken);
await client.messages.create({
body: message,
from: config.from,
to: config.to
});
Logger.info('✔ twilio message sent');
} catch (error) {
Logger.error('✖ couldn\'t send twilio message', error);
}
})();
(async () => {
const givenUrl = link.cartUrl ? link.cartUrl : link.url;
const message = `${Print.inStock(link, store)}\n${givenUrl}`;
try {
await client.messages.create({
body: message,
from: twilio.from,
to: twilio.to
});
logger.info('✔ twilio message sent');
} catch (error) {
logger.error('✖ couldn\'t send twilio message', error);
}
})();
}
}
src/notification/twitter.ts
+17 -13
View File
@@ -1,9 +1,9 @@
import {Link, Store} from '../store/model';
import {Logger, Print} from '../logger';
import {Config} from '../config';
import {Print, logger} from '../logger';
import Twitter from 'twitter';
import {config} from '../config';
const twitter = Config.notifications.twitter;
const twitter = config.notifications.twitter;
const client = new Twitter({
access_token_key: twitter.accessTokenKey,
@@ -13,17 +13,21 @@ const client = new Twitter({
});
export function sendTweet(link: Link, store: Store) {
let status = `${Print.inStock(link, store)}\n${link.cartUrl ? link.cartUrl : link.url}`;
if (twitter.accessTokenKey && twitter.accessTokenSecret && twitter.consumerKey && twitter.consumerSecret) {
logger.debug('↗ sending twitter message');
if (twitter.tweetTags) {
status += `\n\n${twitter.tweetTags}`;
}
let status = `${Print.inStock(link, store)}\n${link.cartUrl ? link.cartUrl : link.url}`;
client.post('statuses/update', {status}, error => {
if (error) {
Logger.error('✖ couldn\'t send twitter notification', error);
} else {
Logger.info('✔ twitter notification sent');
if (twitter.tweetTags) {
status += `\n\n${twitter.tweetTags}`;
}
});
client.post('statuses/update', {status}, error => {
if (error) {
logger.error('✖ couldn\'t send twitter notification', error);
} else {
logger.info('✔ twitter notification sent');
}
});
}
}
src/store/fetch-links.ts
+6 -6
View File
@@ -1,5 +1,5 @@
import {Link, Series, Store} from './model';
import {Logger, Print} from '../logger';
import {Print, logger} from '../logger';
import {Browser} from 'puppeteer';
import cheerio from 'cheerio';
import {filterSeries} from './filter';
@@ -7,7 +7,7 @@ import {usingResponse} from '../util';
function addNewLinks(store: Store, links: Link[], series: Series) {
if (links.length === 0) {
Logger.error(Print.message('NO STORE LINKS FOUND', series, store, true));
logger.error(Print.message('NO STORE LINKS FOUND', series, store, true));
return;
}
@@ -19,8 +19,8 @@ function addNewLinks(store: Store, links: Link[], series: Series) {
return;
}
Logger.info(Print.message(`FOUND ${newLinks.length} STORE LINKS`, series, store, true));
Logger.debug(JSON.stringify(newLinks, null, 2));
logger.info(Print.message(`FOUND ${newLinks.length} STORE LINKS`, series, store, true));
logger.debug(JSON.stringify(newLinks, null, 2));
store.links = store.links.concat(newLinks);
}
@@ -37,13 +37,13 @@ export async function fetchLinks(store: Store, browser: Browser) {
continue;
}
Logger.info(Print.message('DETECTING STORE LINKS', series, store, true));
logger.info(Print.message('DETECTING STORE LINKS', series, store, true));
promises.push(usingResponse(browser, url, async response => {
const text = await response?.text();
if (!text) {
Logger.error(Print.message('NO RESPONSE', series, store, true));
logger.error(Print.message('NO RESPONSE', series, store, true));
return;
}
src/store/filter.ts
+7 -7
View File
@@ -1,5 +1,5 @@
import {Config} from '../config';
import {Link} from './model';
import {config} from '../config';
/**
* Returns true if the brand should be checked for stock
@@ -7,11 +7,11 @@ import {Link} from './model';
* @param brand The brand of the GPU
*/
function filterBrand(brand: Link['brand']): boolean {
if (Config.store.showOnlyBrands.length === 0) {
if (config.store.showOnlyBrands.length === 0) {
return true;
}
return Config.store.showOnlyBrands.includes(brand);
return config.store.showOnlyBrands.includes(brand);
}
/**
@@ -20,12 +20,12 @@ function filterBrand(brand: Link['brand']): boolean {
* @param model The model of the GPU
*/
function filterModel(model: Link['model']): boolean {
if (Config.store.showOnlyModels.length === 0) {
if (config.store.showOnlyModels.length === 0) {
return true;
}
const sanitizedModel = model.replace(/\s/g, '');
for (const configModel of Config.store.showOnlyModels) {
for (const configModel of config.store.showOnlyModels) {
const sanitizedConfigModel = configModel.replace(/\s/g, '');
if (sanitizedModel === sanitizedConfigModel) {
return true;
@@ -41,11 +41,11 @@ function filterModel(model: Link['model']): boolean {
* @param series The series of the GPU
*/
export function filterSeries(series: Link['series']): boolean {
if (Config.store.showOnlySeries.length === 0) {
if (config.store.showOnlySeries.length === 0) {
return true;
}
return Config.store.showOnlySeries.includes(series);
return config.store.showOnlySeries.includes(series);
}
/**
src/store/includes-labels.ts
+3 -3
View File
@@ -1,6 +1,6 @@
import {Element, LabelQuery, Pricing} from './model';
import {Logger} from '../logger';
import {Page} from 'puppeteer';
import {logger} from '../logger';
export type Selector = {
requireVisible: boolean;
@@ -44,7 +44,7 @@ export async function pageIncludesLabels(page: Page, query: LabelQuery, options:
return false;
}
Logger.debug(contents);
logger.debug(contents);
return includesLabels(contents, query.text);
}));
@@ -103,7 +103,7 @@ export async function cardPriceLimit(page: Page, query: Pricing, max: number, op
const priceSeperator = query.euroFormat ? /\./g : /,/g;
const cardpriceNumber = Number.parseFloat(cardPrice.replace(priceSeperator, '').match(/\d+/g)!.join('.'));
Logger.debug(`Raw card price: ${cardPrice} | Limit: ${max}`);
logger.debug(`Raw card price: ${cardPrice} | Limit: ${max}`);
return cardpriceNumber > max ? cardpriceNumber : null;
}
src/store/lookup.ts
+26 -26
View File
@@ -1,9 +1,9 @@
import {Browser, Page, Response} from 'puppeteer';
import {Link, Store} from './model';
import {Logger, Print} from '../logger';
import {Print, logger} from '../logger';
import {Selector, cardPriceLimit, pageIncludesLabels} from './includes-labels';
import {closePage, delay, getSleepTime, isStatusCodeInRange} from '../util';
import {Config} from '../config';
import {config} from '../config';
import {disableBlockerInPage} from '../adblocker';
import {filterStoreLink} from './filter';
import open from 'open';
@@ -27,20 +27,20 @@ async function lookup(browser: Browser, store: Store) {
continue;
}
if (Config.page.inStockWaitTime && inStock[link.url]) {
Logger.info(Print.inStockWaiting(link, store, true));
if (config.page.inStockWaitTime && inStock[link.url]) {
logger.info(Print.inStockWaiting(link, store, true));
continue;
}
const page = await browser.newPage();
page.setDefaultNavigationTimeout(Config.page.navigationTimeout);
await page.setUserAgent(Config.page.userAgent);
page.setDefaultNavigationTimeout(config.page.timeout);
await page.setUserAgent(config.page.userAgent);
if (store.disableAdBlocker) {
try {
await disableBlockerInPage(page);
} catch (error) {
Logger.error(error);
logger.error(error);
}
}
@@ -49,7 +49,7 @@ async function lookup(browser: Browser, store: Store) {
try {
statusCode = await lookupCard(browser, store, page, link);
} catch (error) {
Logger.error(`✖ [${store.name}] ${link.brand} ${link.model} - ${error.message as string}`);
logger.error(`✖ [${store.name}] ${link.brand} ${link.model} - ${error.message as string}`);
}
// Must apply backoff before closing the page, e.g. if CloudFlare is
@@ -67,16 +67,16 @@ async function lookupCard(browser: Browser, store: Store, page: Page, link: Link
const response: Response | null = await page.goto(link.url, {waitUntil: givenWaitFor});
if (!response) {
Logger.debug(Print.noResponse(link, store, true));
logger.debug(Print.noResponse(link, store, true));
}
const successStatusCodes = store.successStatusCodes ?? [[0, 399]];
const statusCode = response?.status() ?? 0;
if (!isStatusCodeInRange(statusCode, successStatusCodes)) {
if (statusCode === 429) {
Logger.warn(Print.rateLimit(link, store, true));
logger.warn(Print.rateLimit(link, store, true));
} else {
Logger.warn(Print.badStatusCode(link, store, statusCode, true));
logger.warn(Print.badStatusCode(link, store, statusCode, true));
}
return statusCode;
@@ -84,9 +84,9 @@ async function lookupCard(browser: Browser, store: Store, page: Page, link: Link
if (await lookupCardInStock(store, page, link)) {
const givenUrl = link.cartUrl ? link.cartUrl : link.url;
Logger.info(`${Print.inStock(link, store, true)}\n${givenUrl}`);
logger.info(`${Print.inStock(link, store, true)}\n${givenUrl}`);
if (Config.browser.open) {
if (config.browser.open) {
if (link.openCartAction === undefined) {
await open(givenUrl);
} else {
@@ -96,16 +96,16 @@ async function lookupCard(browser: Browser, store: Store, page: Page, link: Link
sendNotification(link, store);
if (Config.page.inStockWaitTime) {
if (config.page.inStockWaitTime) {
inStock[link.url] = true;
setTimeout(() => {
inStock[link.url] = false;
}, 1000 * Config.page.inStockWaitTime);
}, 1000 * config.page.inStockWaitTime);
}
if (Config.page.screenshot) {
Logger.debug('ℹ saving screenshot');
if (config.page.screenshot) {
logger.debug('ℹ saving screenshot');
link.screenshot = `success-${Date.now()}.png`;
await page.screenshot({path: link.screenshot});
@@ -126,36 +126,36 @@ async function lookupCardInStock(store: Store, page: Page, link: Link) {
const options = {...baseOptions, requireVisible: true, type: 'outerHTML' as const};
if (!await pageIncludesLabels(page, store.labels.inStock, options)) {
Logger.info(Print.outOfStock(link, store, true));
logger.info(Print.outOfStock(link, store, true));
return false;
}
}
if (store.labels.outOfStock) {
if (await pageIncludesLabels(page, store.labels.outOfStock, baseOptions)) {
Logger.info(Print.outOfStock(link, store, true));
logger.info(Print.outOfStock(link, store, true));
return false;
}
}
if (store.labels.bannedSeller) {
if (await pageIncludesLabels(page, store.labels.bannedSeller, baseOptions)) {
Logger.warn(Print.bannedSeller(link, store, true));
logger.warn(Print.bannedSeller(link, store, true));
return false;
}
}
if (store.labels.maxPrice) {
const priceLimit = await cardPriceLimit(page, store.labels.maxPrice, Config.store.maxPrice, baseOptions);
const priceLimit = await cardPriceLimit(page, store.labels.maxPrice, config.store.maxPrice, baseOptions);
if (priceLimit) {
Logger.info(Print.maxPrice(link, store, priceLimit, true));
logger.info(Print.maxPrice(link, store, priceLimit, true));
return false;
}
}
if (store.labels.captcha) {
if (await pageIncludesLabels(page, store.labels.captcha, baseOptions)) {
Logger.warn(Print.captcha(link, store, true));
logger.warn(Print.captcha(link, store, true));
await delay(getSleepTime());
return false;
}
@@ -165,14 +165,14 @@ async function lookupCardInStock(store: Store, page: Page, link: Link) {
}
export async function tryLookupAndLoop(browser: Browser, store: Store) {
Logger.debug(`[${store.name}] Starting lookup...`);
logger.debug(`[${store.name}] Starting lookup...`);
try {
await lookup(browser, store);
} catch (error) {
Logger.error(error);
logger.error(error);
}
const sleepTime = getSleepTime();
Logger.debug(`[${store.name}] Lookup done, next one in ${sleepTime} ms`);
logger.debug(`[${store.name}] Lookup done, next one in ${sleepTime} ms`);
setTimeout(tryLookupAndLoop, sleepTime, browser, store);
}
src/store/model/helpers/backoff.ts
+6 -6
View File
@@ -1,7 +1,7 @@
import {Link, Store} from '..';
import {Logger, Print} from '../../../logger';
import {Print, logger} from '../../../logger';
import {delay, isStatusCodeInRange} from '../../../util';
import {Config} from '../../../config';
import {config} from '../../../config';
type Backoff = {
count: number;
@@ -27,26 +27,26 @@ export async function processBackoffDelay(store: Store, link: Link, statusCode:
let backoff = stores[store.name];
if (!backoff) {
backoff = {count: 0, time: Config.browser.minBackoff};
backoff = {count: 0, time: config.browser.minBackoff};
stores[store.name] = backoff;
}
if (!isBackoff) {
if (backoff.count > 0) {
backoff.count--;
backoff.time = Math.max(backoff.time / 2, Config.browser.minBackoff);
backoff.time = Math.max(backoff.time / 2, config.browser.minBackoff);
}
return -1;
}
const backoffTime = backoff.time;
Logger.debug(Print.backoff(link, store, {delay: backoffTime, statusCode}, true));
logger.debug(Print.backoff(link, store, {delay: backoffTime, statusCode}, true));
await delay(backoff.time);
backoff.count++;
backoff.time = Math.min(backoff.time * 2, Config.browser.maxBackoff);
backoff.time = Math.min(backoff.time * 2, config.browser.maxBackoff);
return backoffTime;
}
src/store/model/helpers/nvidia-cart.ts
+17 -17
View File
@@ -1,8 +1,8 @@
import {NvidiaRegionInfo, regionInfos} from '../nvidia-api';
import {usingPage, usingResponse} from '../../../util';
import {Browser} from 'puppeteer';
import {Config} from '../../../config';
import {Logger} from '../../../logger';
import {config} from '../../../config';
import {logger} from '../../../logger';
import open from 'open';
interface NvidiaSessionTokenJSON {
@@ -34,7 +34,7 @@ export class NvidiaCart {
await this.refreshSessionToken();
setTimeout(callback, Config.nvidia.sessionTtl);
setTimeout(callback, config.nvidia.sessionTtl);
};
this.isKeepAlive = true;
@@ -47,7 +47,7 @@ export class NvidiaCart {
}
public get regionInfo(): NvidiaRegionInfo {
const country = Config.store.country;
const country = config.store.country;
const regionInfo = regionInfos.get(country);
if (!regionInfo) {
throw new Error(`Unknown country ${country}`);
@@ -62,20 +62,20 @@ export class NvidiaCart {
public async addToCard(productId: number, name: string): Promise<string> {
let cartUrl: string | undefined;
Logger.info(`🚀🚀🚀 [nvidia] ${name}, starting auto add to cart 🚀🚀🚀`);
logger.info(`🚀🚀🚀 [nvidia] ${name}, starting auto add to cart 🚀🚀🚀`);
try {
Logger.info(`🚀🚀🚀 [nvidia] ${name}, adding to cart 🚀🚀🚀`);
logger.info(`🚀🚀🚀 [nvidia] ${name}, adding to cart 🚀🚀🚀`);
let lastError: Error | string | undefined;
/* eslint-disable no-await-in-loop */
for (let i = 0; i < Config.nvidia.addToCardAttempts; i++) {
for (let i = 0; i < config.nvidia.addToCardAttempts; i++) {
try {
cartUrl = await this.addToCartAndGetLocationRedirect(productId);
break;
} catch (error) {
Logger.error(`✖ [nvidia] ${name} could not automatically add to cart, attempt ${i + 1} of ${Config.nvidia.addToCardAttempts}`, error);
Logger.debug(error);
logger.error(`✖ [nvidia] ${name} could not automatically add to cart, attempt ${i + 1} of ${config.nvidia.addToCardAttempts}`, error);
logger.debug(error);
lastError = error;
}
@@ -87,13 +87,13 @@ export class NvidiaCart {
throw lastError;
}
Logger.info(`🚀🚀🚀 [nvidia] ${name}, opening checkout page 🚀🚀🚀`);
Logger.info(cartUrl);
logger.info(`🚀🚀🚀 [nvidia] ${name}, opening checkout page 🚀🚀🚀`);
logger.info(cartUrl);
await open(cartUrl);
} catch (error) {
Logger.error(`✖ [nvidia] ${name} could not automatically add to cart, opening page`);
Logger.debug(error);
logger.error(`✖ [nvidia] ${name} could not automatically add to cart, opening page`);
logger.debug(error);
cartUrl = this.fallbackCartUrl;
@@ -116,7 +116,7 @@ export class NvidiaCart {
}
public async refreshSessionToken(): Promise<void> {
Logger.debug('ℹ [nvidia] refreshing session token');
logger.debug('ℹ [nvidia] refreshing session token');
try {
const result = await usingResponse(this.browser, this.sessionUrl, async response => {
return response?.json() as NvidiaSessionTokenJSON | undefined;
@@ -126,10 +126,10 @@ export class NvidiaCart {
}
this.sessionToken = result.session_token;
Logger.debug(`ℹ [nvidia] session_token=${result.session_token}`);
logger.debug(`ℹ [nvidia] session_token=${result.session_token}`);
} catch (error) {
const message: string = typeof error === 'object' ? error.message : error;
Logger.error(`✖ [nvidia] ${message}`);
logger.error(`✖ [nvidia] ${message}`);
}
}
@@ -137,7 +137,7 @@ export class NvidiaCart {
const url = 'https://api-prod.nvidia.com/direct-sales-shop/DR/add-to-cart';
const sessionToken = await this.getSessionToken();
Logger.info(`ℹ [nvidia] session_token=${sessionToken}`);
logger.info(`ℹ [nvidia] session_token=${sessionToken}`);
const locationData = await usingPage(this.browser, async page => {
page.removeAllListeners('request');
src/store/model/helpers/nvidia.ts
+3 -3
View File
@@ -1,12 +1,12 @@
import {NvidiaRegionInfo, regionInfos} from '../nvidia-api';
import {Browser} from 'puppeteer';
import {Config} from '../../../config';
import {Link} from '../store';
import {NvidiaCart} from './nvidia-cart';
import {config} from '../../../config';
import {timestampUrlParameter} from '../../timestamp-url-parameter';
function getRegionInfo(): NvidiaRegionInfo {
let country = Config.store.country;
let country = config.store.country;
if (!regionInfos.has(country)) {
country = 'usa';
}
@@ -30,7 +30,7 @@ export function generateSetupAction() {
return async (browser: Browser) => {
cart = new NvidiaCart(browser);
if (Config.browser.open) {
if (config.browser.open) {
cart.keepAlive();
}
};
src/store/model/index.ts
+11 -11
View File
@@ -8,11 +8,9 @@ import {AsusDe} from './asus-de';
import {BAndH} from './bandh';
import {BestBuy} from './bestbuy';
import {BestBuyCa} from './bestbuy-ca';
import {Config} from '../../config';
import {Evga} from './evga';
import {EvgaEu} from './evga-eu';
import {Gamestop} from './gamestop';
import {Logger} from '../../logger';
import {MicroCenter} from './microcenter';
import {Newegg} from './newegg';
import {NeweggCa} from './newegg-ca';
@@ -22,6 +20,8 @@ import {OfficeDepot} from './officedepot';
import {Pny} from './pny';
import {Store} from './store';
import {Zotac} from './zotac';
import {config} from '../../config';
import {logger} from '../../logger';
const masterList = new Map([
[Adorama.name, Adorama],
@@ -49,27 +49,27 @@ const masterList = new Map([
const list = new Map();
for (const name of Config.store.stores) {
for (const name of config.store.stores) {
if (masterList.has(name)) {
list.set(name, masterList.get(name));
} else {
const logString = `No store named ${name}, skipping.`;
Logger.warn(logString);
logger.warn(logString);
}
}
Logger.info(`ℹ selected stores: ${Array.from(list.keys()).join(', ')}`);
logger.info(`ℹ selected stores: ${Array.from(list.keys()).join(', ')}`);
if (Config.store.showOnlyBrands.length > 0) {
Logger.info(`ℹ selected brands: ${Config.store.showOnlyBrands.join(', ')}`);
if (config.store.showOnlyBrands.length > 0) {
logger.info(`ℹ selected brands: ${config.store.showOnlyBrands.join(', ')}`);
}
if (Config.store.showOnlyModels.length > 0) {
Logger.info(`ℹ selected models: ${Config.store.showOnlyModels.join(', ')}`);
if (config.store.showOnlyModels.length > 0) {
logger.info(`ℹ selected models: ${config.store.showOnlyModels.join(', ')}`);
}
if (Config.store.showOnlySeries.length > 0) {
Logger.info(`ℹ selected series: ${Config.store.showOnlySeries.join(', ')}`);
if (config.store.showOnlySeries.length > 0) {
logger.info(`ℹ selected series: ${config.store.showOnlySeries.join(', ')}`);
}
export const Stores = Array.from(list.values()) as Store[];
src/store/model/microcenter.ts
+5 -4
View File
@@ -1,7 +1,8 @@
import {Config} from '../../config';
import {Store} from './store';
import {config} from '../../config';
const microCenterLocation = config.store.microCenterLocation;
const MicroCenterLocation = Config.store.microCenterLocation;
const microCenterLocationToId: Map<string, string> = new Map([
['web', '029'],
['brooklyn', '115'],
@@ -32,10 +33,10 @@ const microCenterLocationToId: Map<string, string> = new Map([
]);
let storeId: string;
if (microCenterLocationToId.get(MicroCenterLocation) === undefined) {
if (microCenterLocationToId.get(microCenterLocation) === undefined) {
storeId = '029';
} else {
storeId = microCenterLocationToId.get(MicroCenterLocation)!;
storeId = microCenterLocationToId.get(microCenterLocation)!;
}
export const MicroCenter: Store = {
src/types/pushbullet.d.ts
Vendored
+1 -1
View File
@@ -1 +1 @@
declare module 'pushbullet';
declare module '@hijef/pushbullet';
src/util.ts
+7 -7
View File
@@ -1,11 +1,11 @@
import {Browser, Page, Response} from 'puppeteer';
import {Config} from './config';
import {Logger} from './logger';
import {StatusCodeRangeArray} from './store/model';
import {config} from './config';
import {disableBlockerInPage} from './adblocker';
import {logger} from './logger';
export function getSleepTime() {
return Config.browser.minSleep + (Math.random() * (Config.browser.maxSleep - Config.browser.minSleep));
return config.browser.minSleep + (Math.random() * (config.browser.maxSleep - config.browser.minSleep));
}
export async function delay(ms: number) {
@@ -47,8 +47,8 @@ export async function usingResponse<T>(
export async function usingPage<T>(browser: Browser, cb: (page: Page, browser: Browser) => Promise<T>): Promise<T> {
const page = await browser.newPage();
page.setDefaultNavigationTimeout(Config.page.navigationTimeout);
await page.setUserAgent(Config.page.userAgent);
page.setDefaultNavigationTimeout(config.page.timeout);
await page.setUserAgent(config.page.userAgent);
try {
return await cb(page, browser);
@@ -56,13 +56,13 @@ export async function usingPage<T>(browser: Browser, cb: (page: Page, browser: B
try {
await closePage(page);
} catch (error) {
Logger.error(error);
logger.error(error);
}
}
}
export async function closePage(page: Page) {
if (!Config.browser.lowBandwidth) {
if (!config.browser.lowBandwidth) {
await disableBlockerInPage(page);
}