From b9b6b55c29d11f48b683816e5b8c1cab127ed5fd Mon Sep 17 00:00:00 2001 From: Andrew Mackrodt Date: Tue, 6 Oct 2020 16:35:54 +0100 Subject: [PATCH] feat: add uk stores (#455) --- README.md | 10 ++++ src/index.ts | 8 --- src/store/fetch-links.ts | 16 +++--- src/store/lookup.ts | 16 ++++++ src/store/model/amazon-uk.ts | 95 +++++++++++++++++++++++++++++++++ src/store/model/aria.ts | 42 +++++++++++++++ src/store/model/box.ts | 45 ++++++++++++++++ src/store/model/ccl.ts | 47 ++++++++++++++++ src/store/model/currys.ts | 43 +++++++++++++++ src/store/model/ebuyer.ts | 47 ++++++++++++++++ src/store/model/helpers/card.ts | 94 +++++++++++++++++++++++++++++--- src/store/model/index.ts | 20 +++++++ src/store/model/novatech.ts | 48 +++++++++++++++++ src/store/model/overclockers.ts | 51 ++++++++++++++++++ src/store/model/scan.ts | 52 ++++++++++++++++++ src/store/model/store.ts | 3 +- src/store/model/very.ts | 79 +++++++++++++++++++++++++++ 17 files changed, 695 insertions(+), 21 deletions(-) create mode 100644 src/store/model/amazon-uk.ts create mode 100644 src/store/model/aria.ts create mode 100644 src/store/model/box.ts create mode 100644 src/store/model/ccl.ts create mode 100644 src/store/model/currys.ts create mode 100644 src/store/model/ebuyer.ts create mode 100644 src/store/model/novatech.ts create mode 100644 src/store/model/overclockers.ts create mode 100644 src/store/model/scan.ts create mode 100644 src/store/model/very.ts diff --git a/README.md b/README.md index 25a6dd1..68aa589 100644 --- a/README.md +++ b/README.md @@ -135,20 +135,30 @@ Here is a list of variables that you can use to customize your newly copied `.en | Amazon (CA) | `amazon-ca`| | Amazon (DE) | `amazon-de`| | Amazon (NL) | `amazon-nl`| +| Amazon (UK) | `amazon-uk`| +| Aria PC | `aria`| | ASUS | `asus` | | B&H | `bandh`| | Best Buy | `bestbuy`| | Best Buy (CA) | `bestbuy-ca`| +| Box | `box`| +| CCL | `ccl`| +| Currys | `currys`| +| eBuyer | `ebuyer`| | EVGA | `evga`| | EVGA (EU) | `evga-eu`| | Gamestop | `gamestop`| | Micro Center | `microcenter`| | Newegg | `newegg`| | Newegg (CA) | `newegg-ca`| +| Novatech | `novatech`| | Nvidia | `nvidia`| | Nvidia (API) | `nvidia-api`| | Office Depot | `officedepot`| +| Overclockers | `overclockers`| | PNY | `pny`| +| Scan | `scan`| +| Very | `very`| | Zotac | `zotac`|
diff --git a/src/index.ts b/src/index.ts index ac7d693..554b71c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,7 +1,6 @@ import {Stores} from './store/model'; import {adBlocker} from './adblocker'; import {config} from './config'; -import {fetchLinks} from './store/fetch-links'; import {getSleepTime} from './util'; import {logger} from './logger'; import puppeteer from 'puppeteer-extra'; @@ -50,21 +49,14 @@ async function main() { headless: config.browser.isHeadless }); - const promises = []; for (const store of Stores) { logger.debug('store links', {meta: {links: store.links}}); if (store.setupAction !== undefined) { store.setupAction(browser); } - if (store.linksBuilder) { - promises.push(fetchLinks(store, browser)); - } - setTimeout(tryLookupAndLoop, getSleepTime(), browser, store); } - - await Promise.all(promises); } /** diff --git a/src/store/fetch-links.ts b/src/store/fetch-links.ts index 0064d23..8df5341 100644 --- a/src/store/fetch-links.ts +++ b/src/store/fetch-links.ts @@ -7,7 +7,7 @@ import {usingResponse} from '../util'; function addNewLinks(store: Store, links: Link[], series: Series) { if (links.length === 0) { - logger.error(Print.message('NO STORE LINKS FOUND', series, store, true)); + logger.warn(Print.message('NO STORE LINKS FOUND', series, store, true)); return; } @@ -30,16 +30,20 @@ export async function fetchLinks(store: Store, browser: Browser) { return; } - const promises = []; + const promises: Array> = []; - for (const {series, url} of store.linksBuilder.urls) { + for (let {series, url} of store.linksBuilder.urls) { if (!filterSeries(series)) { continue; } - logger.info(Print.message('DETECTING STORE LINKS', series, store, true)); + logger.debug(Print.message('DETECTING STORE LINKS', series, store, true)); - promises.push(usingResponse(browser, url, async response => { + if (!Array.isArray(url)) { + url = [url]; + } + + url.map(x => promises.push(usingResponse(browser, x, async response => { const text = await response?.text(); if (!text) { @@ -51,7 +55,7 @@ export async function fetchLinks(store: Store, browser: Browser) { const links = store.linksBuilder!.builder(docElement, series); addNewLinks(store, links, series); - })); + }))); } await Promise.all(promises); diff --git a/src/store/lookup.ts b/src/store/lookup.ts index 7bc9deb..4c58673 100644 --- a/src/store/lookup.ts +++ b/src/store/lookup.ts @@ -5,6 +5,7 @@ import {Selector, cardPriceLimit, pageIncludesLabels} from './includes-labels'; import {closePage, delay, getSleepTime, isStatusCodeInRange} from '../util'; import {config} from '../config'; import {disableBlockerInPage} from '../adblocker'; +import {fetchLinks} from './fetch-links'; import {filterStoreLink} from './filter'; import open from 'open'; import {processBackoffDelay} from './model/helpers/backoff'; @@ -12,6 +13,8 @@ import {sendNotification} from '../notification'; const inStock: Record = {}; +const linkBuilderLastRunTimes: Record = {}; + /** * Responsible for looking up information about a each product within * a `Store`. It's important that we ignore `no-await-in-loop` here @@ -165,6 +168,19 @@ async function lookupCardInStock(store: Store, page: Page, link: Link) { } export async function tryLookupAndLoop(browser: Browser, store: Store) { + if (store.linksBuilder) { + const lastRunTime = linkBuilderLastRunTimes[store.name] ?? -1; + const ttl = store.linksBuilder.ttl ?? Number.MAX_SAFE_INTEGER; + if (lastRunTime === -1 || (Date.now() - lastRunTime) > ttl) { + try { + await fetchLinks(store, browser); + linkBuilderLastRunTimes[store.name] = Date.now(); + } catch (error) { + logger.error(error.message); + } + } + } + logger.debug(`[${store.name}] Starting lookup...`); try { await lookup(browser, store); diff --git a/src/store/model/amazon-uk.ts b/src/store/model/amazon-uk.ts new file mode 100644 index 0000000..ef7aa91 --- /dev/null +++ b/src/store/model/amazon-uk.ts @@ -0,0 +1,95 @@ +import {Link, Store} from './store'; +import {logger} from '../../logger'; +import {parseCard} from './helpers/card'; + +export const AmazonUk: Store = { + backoffStatusCodes: [403, 429, 503], + labels: { + captcha: { + container: 'body', + text: ['enter the characters you see below'] + }, + inStock: { + container: '#availability', + text: ['in stock'] + }, + maxPrice: { + container: 'span[class*="PriceString"]' + }, + outOfStock: [ + { + container: '#availability', + text: ['out of stock', 'unavailable'] + }, + { + container: '#backInStock', + text: ['unavailable'] + } + ] + }, + links: [ + { + brand: 'test:brand', + cartUrl: 'https://www.amazon.co.uk/gp/aws/cart/add.html?ASIN.1=B081265T5Z&Quantity.1=1', + model: 'test:model', + series: 'test:series', + url: 'https://www.amazon.co.uk/dp/B081265T5Z/' + } + ], + linksBuilder: { + builder: (docElement, series) => { + const productElements = docElement.find('.s-result-list .s-result-item[data-asin]'); + const links: Link[] = []; + for (let i = 0; i < productElements.length; i++) { + const productElement = productElements.eq(i); + const asin = productElement.attr()['data-asin']; + + if (!asin) { + continue; + } + + const url = `https://www.amazon.co.uk/dp/${asin}/`; + const titleElement = productElement.find('.sg-col-inner h2 a.a-text-normal[href] span').first(); + const title = titleElement.text().trim(); + + if (!title || !new RegExp(`RTX.*${series}`, 'i').exec(title)) { + continue; + } + + const card = parseCard(title); + + if (card) { + links.push({ + brand: card.brand as any, + cartUrl: `https://www.amazon.co.uk/gp/aws/cart/add.html?ASIN.1=${asin}&Quantity.1=1`, + model: card.model, + series, + url + }); + } else { + logger.error(`Failed to parse card: ${title}`); + } + } + + return links; + }, + ttl: 300000, + urls: [ + { + series: '3080', + url: [ + 'https://www.amazon.co.uk/s?k=%2B%22RTX+3080%22+-2080+-GTX&i=computers&rh=n%3A430500031%2Cp_n_availability%3A419162031&s=relevancerank&dc&qid=1601675291', + 'https://www.amazon.co.uk/s?k=%2B%22RTX+3080%22+-2080+-GTX&i=computers&rh=n%3A430500031%2Cp_n_availability%3A419162031&s=relevancerank&dc&qid=1601675594&page=2' + ] + }, + { + series: '3090', + url: [ + 'https://www.amazon.co.uk/s?k=%2B%22RTX+3090%22+-3080+-GTX&i=computers&rh=n%3A430500031%2Cp_n_availability%3A419162031&s=relevancerank&dc&qid=1601675291', + 'https://www.amazon.co.uk/s?k=%2B%22RTX+3090%22+-3080+-GTX&i=computers&rh=n%3A430500031%2Cp_n_availability%3A419162031&s=relevancerank&dc&qid=1601675594&page=2' + ] + } + ] + }, + name: 'amazon-uk' +}; diff --git a/src/store/model/aria.ts b/src/store/model/aria.ts new file mode 100644 index 0000000..4b92106 --- /dev/null +++ b/src/store/model/aria.ts @@ -0,0 +1,42 @@ +import {Store} from './store'; +import {getProductLinksBuilder} from './helpers/card'; + +export const Aria: Store = { + labels: { + inStock: { + container: '#addQuantity', + text: ['add to shopping basket'] + }, + outOfStock: { + container: '.fBox', + text: ['out of stock', 'there is currently no stock of this item'] + } + }, + links: [ + { + brand: 'test:brand', + model: 'CARD', + series: 'test:series', + url: 'https://www.aria.co.uk/Products/Components/Graphics+Cards/NVIDIA+GeForce/GeForce+RTX+2060+Super/Gigabyte+NVIDIA+GeForce+RTX+2060+SUPER+8GB+WINDFORCE+OC+Turing+Graphics+Card+%2B+RTX+Bundle%21?productId=71541' + } + ], + linksBuilder: { + builder: getProductLinksBuilder({ + productsSelector: '#productListingInner .listTable .listTableTr', + sitePrefix: 'https://www.aria.co.uk', + titleSelector: 'strong > a[href]' + }), + urls: [ + { + series: '3080', + url: 'https://www.aria.co.uk/Products/Components/Graphics+Cards/NVIDIA+GeForce/GeForce+RTX+3080' + }, + { + series: '3090', + url: 'https://www.aria.co.uk/Products/Components/Graphics+Cards/NVIDIA+GeForce/GeForce+RTX+3090' + } + ] + }, + name: 'aria', + waitUntil: 'domcontentloaded' +}; diff --git a/src/store/model/box.ts b/src/store/model/box.ts new file mode 100644 index 0000000..643e017 --- /dev/null +++ b/src/store/model/box.ts @@ -0,0 +1,45 @@ +import {Store} from './store'; +import {getProductLinksBuilder} from './helpers/card'; + +export const Box: Store = { + labels: { + inStock: { + container: '#divBuyButton', + text: ['add to basket'] + }, + outOfStock: { + text: ['request stock alert', 'coming soon'] + } + }, + links: [ + { + brand: 'test:brand', + model: 'CARD', + series: 'test:series', + url: 'https://www.box.co.uk/ASUS-TUF-GeForce-RTX-2060-6GB-Gaming-Gra_2669497.html' + } + ], + linksBuilder: { + builder: getProductLinksBuilder({ + productsSelector: '.products-right .p-list', + sitePrefix: 'https://www.box.co.uk', + titleSelector: '.p-list-section > h3 > a[href]' + }), + urls: [ + { + series: '3070', + url: 'https://www.box.co.uk/rtx-3070-graphics-cards' + }, + { + series: '3080', + url: 'https://www.box.co.uk/rtx-3080-graphics-cards' + }, + { + series: '3090', + url: 'https://www.box.co.uk/rtx-3090-graphics-cards' + } + ] + }, + name: 'box', + waitUntil: 'domcontentloaded' +}; diff --git a/src/store/model/ccl.ts b/src/store/model/ccl.ts new file mode 100644 index 0000000..72a1e5f --- /dev/null +++ b/src/store/model/ccl.ts @@ -0,0 +1,47 @@ +import {Store} from './store'; +import {getProductLinksBuilder} from './helpers/card'; + +export const Ccl: Store = { + labels: { + inStock: { + container: '#pnlAddToBasket', + text: ['add to basket'] + }, + outOfStock: { + container: '#pnlSoldOut', + text: ['sold out', 'coming soon'] + } + }, + links: [ + { + brand: 'test:brand', + model: 'CARD', + series: 'test:series', + url: 'https://www.cclonline.com/product/296443/RTX-2060-SUPER-VENTUS-GP-OC/Graphics-Cards/MSI-GeForce-RTX-2060-SUPER-VENTUS-GP-OC-8GB-Overclocked-Graphics-Card/VGA5671/' + } + ], + linksBuilder: { + builder: getProductLinksBuilder({ + productsSelector: '.productListingContainerOuter .productList', + sitePrefix: 'https://www.cclonline.com', + titleAttribute: 'title', + titleSelector: '.productList_Detail a[title]' + }), + urls: [ + { + series: '3070', + url: 'https://www.cclonline.com/category/430/PC-Components/Graphics-Cards/GeForce-RTX-3070-Graphics-Cards/' + }, + { + series: '3080', + url: 'https://www.cclonline.com/category/430/PC-Components/Graphics-Cards/GeForce-RTX-3080-Graphics-Cards/' + }, + { + series: '3090', + url: 'https://www.cclonline.com/category/430/PC-Components/Graphics-Cards/GeForce-RTX-3090-Graphics-Cards/' + } + ] + }, + name: 'ccl', + waitUntil: 'domcontentloaded' +}; diff --git a/src/store/model/currys.ts b/src/store/model/currys.ts new file mode 100644 index 0000000..326853c --- /dev/null +++ b/src/store/model/currys.ts @@ -0,0 +1,43 @@ +import {Store} from './store'; +import {getProductLinksBuilder} from './helpers/card'; + +export const Currys: Store = { + labels: { + inStock: { + container: '#product-actions button', + text: ['add to basket'] + }, + outOfStock: { + container: '#product-actions .unavailable', + text: ['not available for delivery'] + } + }, + links: [ + { + brand: 'test:brand', + model: 'CARD', + series: 'test:series', + url: 'https://www.currys.co.uk/gbuk/computing-accessories/components-upgrades/graphics-cards/msi-geforce-rtx-2060-8-gb-super-ventus-gp-oc-graphics-card-10196803-pdt.html' + } + ], + linksBuilder: { + builder: getProductLinksBuilder({ + productsSelector: '.resultList .product', + sitePrefix: 'https://www.currys.co.uk', + titleSelector: '.productTitle', + urlSelector: 'a[href]' + }), + urls: [ + { + series: '3080', + url: 'https://www.currys.co.uk/gbuk/rtx-3080/components-upgrades/graphics-cards/324_3091_30343_xx_ba00013562-bv00313767/xx-criteria.html' + }, + { + series: '3090', + url: 'https://www.currys.co.uk/gbuk/rtx-3090/components-upgrades/graphics-cards/324_3091_30343_xx_ba00013562-bv00313725/xx-criteria.html' + } + ] + }, + name: 'currys', + waitUntil: 'domcontentloaded' +}; diff --git a/src/store/model/ebuyer.ts b/src/store/model/ebuyer.ts new file mode 100644 index 0000000..0036ede --- /dev/null +++ b/src/store/model/ebuyer.ts @@ -0,0 +1,47 @@ +import {Store} from './store'; +import {getProductLinksBuilder} from './helpers/card'; + +export const Ebuyer: Store = { + labels: { + inStock: { + container: '.purchase-info', + text: ['add to basket', 'in stock'] + }, + outOfStock: { + container: '.purchase-info', + text: ['coming soon', 'we are expecting this item on'] + } + }, + links: [ + { + brand: 'test:brand', + model: 'CARD', + series: 'test:series', + url: 'https://www.ebuyer.com/874209-gigabyte-geforce-rtx-2060-windforce-6gb-oc-graphics-card-gv-n2060wf2oc-6gd-v2' + } + ], + linksBuilder: { + builder: getProductLinksBuilder({ + productsSelector: '#list-view .listing-product', + sitePrefix: 'https://www.ebuyer.com', + titleSelector: '.listing-product-title', + urlSelector: 'a[href]' + }), + urls: [ + { + series: '3070', + url: 'https://www.ebuyer.com/store/Components/cat/Graphics-Cards-Nvidia/subcat/GeForce-RTX-3070' + }, + { + series: '3080', + url: 'https://www.ebuyer.com/store/Components/cat/Graphics-Cards-Nvidia/subcat/GeForce-RTX-3080' + }, + { + series: '3090', + url: 'https://www.ebuyer.com/store/Components/cat/Graphics-Cards-Nvidia/subcat/GeForce-RTX-3090' + } + ] + }, + name: 'ebuyer', + waitUntil: 'domcontentloaded' +}; diff --git a/src/store/model/helpers/card.ts b/src/store/model/helpers/card.ts index 5015265..ac3beb0 100644 --- a/src/store/model/helpers/card.ts +++ b/src/store/model/helpers/card.ts @@ -1,13 +1,91 @@ +import {Link, Series} from '../store'; +import {logger} from '../../../logger'; + export interface Card { brand: string; model: string; } +interface LinksBuilderOptions { + productsSelector: string; + sitePrefix: string; + titleAttribute?: string; + titleSelector: string; + urlSelector?: string; +} + +const isPartialUrlRegExp = /^\/[^/]/i; + +export function getProductLinksBuilder(options: LinksBuilderOptions) { + /* eslint-disable unicorn/no-fn-reference-in-iterator */ + return (docElement: cheerio.Cheerio, series: Series): Link[] => { + const productElements = docElement.find(options.productsSelector); + const links: Link[] = []; + for (let i = 0; i < productElements.length; i++) { + const productElement = productElements.eq(i); + const titleElement = productElement.find(options.titleSelector).first(); + let title: string; + + if (options.titleAttribute) { + title = titleElement.attr()?.[options.titleAttribute]; + } else { + title = titleElement.text()?.replace(/\n/g, ' ').trim(); + } + + if (!title) { + continue; + } + + let urlElement = titleElement; + + if (options.urlSelector) { + urlElement = urlElement.find(options.urlSelector).first(); + } + + let url = urlElement.attr()?.href; + + if (!url) { + continue; + } + + if (isPartialUrlRegExp.exec(url)) { + url = options.sitePrefix + url; + } + + const card = parseCard(title); + + if (card) { + links.push({ + brand: card.brand as any, + model: card.model, + series, + url + }); + } else { + logger.error(`Failed to parse card: ${title}`); + } + } + + return links; + }; + /* eslint-enable unicorn/no-fn-reference-in-iterator */ +} + export function parseCard(name: string): Card | null { - name = name.replace(/[^\w ]+/g, '').trim(); - name = name.replace(/\bgraphics card\b/gi, '').trim(); - name = name.replace(/\b\w+ fan\b/gi, '').trim(); - name = name.replace(/\s{2,}/g, ' '); + name = name.replace(/\w+-\w+-[^ ]+/g, ''); + name = name.replace(/\([^(]*\)/g, ''); + name = name.replace(/, .+$/, ''); + name = name.replace(/ with .+$/, ''); + + // Account for incorrect titles, e.g. MSIGeforce + name = name.replace(/geforce/i, ''); + + name = name.replace(/[^\w ]+/g, ''); + name = name.replace(/\bgraphics card\b/gi, ''); + name = name.replace(/\b(? Link[]; - urls: Array<{series: Series; url: string}>; + ttl?: number; + urls: Array<{series: Series; url: string | string[]}>; }; labels: Labels; name: string; diff --git a/src/store/model/very.ts b/src/store/model/very.ts new file mode 100644 index 0000000..42bda8b --- /dev/null +++ b/src/store/model/very.ts @@ -0,0 +1,79 @@ +import {Link, Store} from './store'; +import {logger} from '../../logger'; +import {parseCard} from './helpers/card'; + +export const Very: Store = { + labels: { + inStock: { + container: '.stockMessaging .indicator', + text: ['available', 'low stock'] + }, + outOfStock: { + container: '.stockMessaging .indicator', + text: ['pre-order'] + } + }, + links: [ + { + brand: 'test:brand', + model: 'CARD', + series: 'test:series', + url: 'https://www.very.co.uk/msi-geforce-rtx-2060-super-ventus-gp-oc/1600463772.prd' + } + ], + linksBuilder: { + builder: (docElement, series) => { + const productElements = docElement.find('.productList .product'); + const links: Link[] = []; + for (let i = 0; i < productElements.length; i++) { + const productElement = productElements.eq(i); + const titleElement = productElement.find('.productTitle').first(); + const title = titleElement.text()?.replace(/\n/g, ' ').trim(); + + if (!title || ['RTX', series] + .map(x => title.toLowerCase().includes(x.toLowerCase())) + .filter(x => !x).length > 0 + ) { + continue; + } + + const url = titleElement.attr()?.href; + + if (!url) { + continue; + } + + const card = parseCard(title); + + if (card) { + links.push({ + brand: card.brand as any, + model: card.model, + series, + url + }); + } else { + logger.error(`Failed to parse card: ${title}`); + } + } + + return links; + }, + ttl: 300000, + urls: [ + { + series: '3070', + url: 'https://www.very.co.uk/electricals/pc-components/graphics-cards/e/b/118786.end?sort=newin,0&numProducts=100' + }, + { + series: '3080', + url: 'https://www.very.co.uk/electricals/pc-components/graphics-cards/e/b/118786.end?sort=newin,0&numProducts=100' + }, + { + series: '3090', + url: 'https://www.very.co.uk/electricals/pc-components/graphics-cards/e/b/118786.end?sort=newin,0&numProducts=100' + } + ] + }, + name: 'very' +};