mirror of
https://github.com/opelly27/streetmerchant.git
synced 2026-05-20 02:57:34 +00:00
feat: add uk stores (#455)
This commit is contained in:
Andrew Mackrodt
@@ -135,20 +135,30 @@ Here is a list of variables that you can use to customize your newly copied `.en
|
||||
| Amazon (CA) | `amazon-ca`|
|
||||
| Amazon (DE) | `amazon-de`|
|
||||
| Amazon (NL) | `amazon-nl`|
|
||||
| Amazon (UK) | `amazon-uk`|
|
||||
| Aria PC | `aria`|
|
||||
| ASUS | `asus` |
|
||||
| B&H | `bandh`|
|
||||
| Best Buy | `bestbuy`|
|
||||
| Best Buy (CA) | `bestbuy-ca`|
|
||||
| Box | `box`|
|
||||
| CCL | `ccl`|
|
||||
| Currys | `currys`|
|
||||
| eBuyer | `ebuyer`|
|
||||
| EVGA | `evga`|
|
||||
| EVGA (EU) | `evga-eu`|
|
||||
| Gamestop | `gamestop`|
|
||||
| Micro Center | `microcenter`|
|
||||
| Newegg | `newegg`|
|
||||
| Newegg (CA) | `newegg-ca`|
|
||||
| Novatech | `novatech`|
|
||||
| Nvidia | `nvidia`|
|
||||
| Nvidia (API) | `nvidia-api`|
|
||||
| Office Depot | `officedepot`|
|
||||
| Overclockers | `overclockers`|
|
||||
| PNY | `pny`|
|
||||
| Scan | `scan`|
|
||||
| Very | `very`|
|
||||
| Zotac | `zotac`|
|
||||
|
||||
<details>
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import {Stores} from './store/model';
|
||||
import {adBlocker} from './adblocker';
|
||||
import {config} from './config';
|
||||
import {fetchLinks} from './store/fetch-links';
|
||||
import {getSleepTime} from './util';
|
||||
import {logger} from './logger';
|
||||
import puppeteer from 'puppeteer-extra';
|
||||
@@ -50,21 +49,14 @@ async function main() {
|
||||
headless: config.browser.isHeadless
|
||||
});
|
||||
|
||||
const promises = [];
|
||||
for (const store of Stores) {
|
||||
logger.debug('store links', {meta: {links: store.links}});
|
||||
if (store.setupAction !== undefined) {
|
||||
store.setupAction(browser);
|
||||
}
|
||||
|
||||
if (store.linksBuilder) {
|
||||
promises.push(fetchLinks(store, browser));
|
||||
}
|
||||
|
||||
setTimeout(tryLookupAndLoop, getSleepTime(), browser, store);
|
||||
}
|
||||
|
||||
await Promise.all(promises);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -7,7 +7,7 @@ import {usingResponse} from '../util';
|
||||
|
||||
function addNewLinks(store: Store, links: Link[], series: Series) {
|
||||
if (links.length === 0) {
|
||||
logger.error(Print.message('NO STORE LINKS FOUND', series, store, true));
|
||||
logger.warn(Print.message('NO STORE LINKS FOUND', series, store, true));
|
||||
|
||||
return;
|
||||
}
|
||||
@@ -30,16 +30,20 @@ export async function fetchLinks(store: Store, browser: Browser) {
|
||||
return;
|
||||
}
|
||||
|
||||
const promises = [];
|
||||
const promises: Array<Promise<void>> = [];
|
||||
|
||||
for (const {series, url} of store.linksBuilder.urls) {
|
||||
for (let {series, url} of store.linksBuilder.urls) {
|
||||
if (!filterSeries(series)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
logger.info(Print.message('DETECTING STORE LINKS', series, store, true));
|
||||
logger.debug(Print.message('DETECTING STORE LINKS', series, store, true));
|
||||
|
||||
promises.push(usingResponse(browser, url, async response => {
|
||||
if (!Array.isArray(url)) {
|
||||
url = [url];
|
||||
}
|
||||
|
||||
url.map(x => promises.push(usingResponse(browser, x, async response => {
|
||||
const text = await response?.text();
|
||||
|
||||
if (!text) {
|
||||
@@ -51,7 +55,7 @@ export async function fetchLinks(store: Store, browser: Browser) {
|
||||
const links = store.linksBuilder!.builder(docElement, series);
|
||||
|
||||
addNewLinks(store, links, series);
|
||||
}));
|
||||
})));
|
||||
}
|
||||
|
||||
await Promise.all(promises);
|
||||
|
||||
@@ -5,6 +5,7 @@ import {Selector, cardPriceLimit, pageIncludesLabels} from './includes-labels';
|
||||
import {closePage, delay, getSleepTime, isStatusCodeInRange} from '../util';
|
||||
import {config} from '../config';
|
||||
import {disableBlockerInPage} from '../adblocker';
|
||||
import {fetchLinks} from './fetch-links';
|
||||
import {filterStoreLink} from './filter';
|
||||
import open from 'open';
|
||||
import {processBackoffDelay} from './model/helpers/backoff';
|
||||
@@ -12,6 +13,8 @@ import {sendNotification} from '../notification';
|
||||
|
||||
const inStock: Record<string, boolean> = {};
|
||||
|
||||
const linkBuilderLastRunTimes: Record<string, number> = {};
|
||||
|
||||
/**
|
||||
* Responsible for looking up information about a each product within
|
||||
* a `Store`. It's important that we ignore `no-await-in-loop` here
|
||||
@@ -165,6 +168,19 @@ async function lookupCardInStock(store: Store, page: Page, link: Link) {
|
||||
}
|
||||
|
||||
export async function tryLookupAndLoop(browser: Browser, store: Store) {
|
||||
if (store.linksBuilder) {
|
||||
const lastRunTime = linkBuilderLastRunTimes[store.name] ?? -1;
|
||||
const ttl = store.linksBuilder.ttl ?? Number.MAX_SAFE_INTEGER;
|
||||
if (lastRunTime === -1 || (Date.now() - lastRunTime) > ttl) {
|
||||
try {
|
||||
await fetchLinks(store, browser);
|
||||
linkBuilderLastRunTimes[store.name] = Date.now();
|
||||
} catch (error) {
|
||||
logger.error(error.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logger.debug(`[${store.name}] Starting lookup...`);
|
||||
try {
|
||||
await lookup(browser, store);
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
import {Link, Store} from './store';
|
||||
import {logger} from '../../logger';
|
||||
import {parseCard} from './helpers/card';
|
||||
|
||||
export const AmazonUk: Store = {
|
||||
backoffStatusCodes: [403, 429, 503],
|
||||
labels: {
|
||||
captcha: {
|
||||
container: 'body',
|
||||
text: ['enter the characters you see below']
|
||||
},
|
||||
inStock: {
|
||||
container: '#availability',
|
||||
text: ['in stock']
|
||||
},
|
||||
maxPrice: {
|
||||
container: 'span[class*="PriceString"]'
|
||||
},
|
||||
outOfStock: [
|
||||
{
|
||||
container: '#availability',
|
||||
text: ['out of stock', 'unavailable']
|
||||
},
|
||||
{
|
||||
container: '#backInStock',
|
||||
text: ['unavailable']
|
||||
}
|
||||
]
|
||||
},
|
||||
links: [
|
||||
{
|
||||
brand: 'test:brand',
|
||||
cartUrl: 'https://www.amazon.co.uk/gp/aws/cart/add.html?ASIN.1=B081265T5Z&Quantity.1=1',
|
||||
model: 'test:model',
|
||||
series: 'test:series',
|
||||
url: 'https://www.amazon.co.uk/dp/B081265T5Z/'
|
||||
}
|
||||
],
|
||||
linksBuilder: {
|
||||
builder: (docElement, series) => {
|
||||
const productElements = docElement.find('.s-result-list .s-result-item[data-asin]');
|
||||
const links: Link[] = [];
|
||||
for (let i = 0; i < productElements.length; i++) {
|
||||
const productElement = productElements.eq(i);
|
||||
const asin = productElement.attr()['data-asin'];
|
||||
|
||||
if (!asin) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const url = `https://www.amazon.co.uk/dp/${asin}/`;
|
||||
const titleElement = productElement.find('.sg-col-inner h2 a.a-text-normal[href] span').first();
|
||||
const title = titleElement.text().trim();
|
||||
|
||||
if (!title || !new RegExp(`RTX.*${series}`, 'i').exec(title)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const card = parseCard(title);
|
||||
|
||||
if (card) {
|
||||
links.push({
|
||||
brand: card.brand as any,
|
||||
cartUrl: `https://www.amazon.co.uk/gp/aws/cart/add.html?ASIN.1=${asin}&Quantity.1=1`,
|
||||
model: card.model,
|
||||
series,
|
||||
url
|
||||
});
|
||||
} else {
|
||||
logger.error(`Failed to parse card: ${title}`);
|
||||
}
|
||||
}
|
||||
|
||||
return links;
|
||||
},
|
||||
ttl: 300000,
|
||||
urls: [
|
||||
{
|
||||
series: '3080',
|
||||
url: [
|
||||
'https://www.amazon.co.uk/s?k=%2B%22RTX+3080%22+-2080+-GTX&i=computers&rh=n%3A430500031%2Cp_n_availability%3A419162031&s=relevancerank&dc&qid=1601675291',
|
||||
'https://www.amazon.co.uk/s?k=%2B%22RTX+3080%22+-2080+-GTX&i=computers&rh=n%3A430500031%2Cp_n_availability%3A419162031&s=relevancerank&dc&qid=1601675594&page=2'
|
||||
]
|
||||
},
|
||||
{
|
||||
series: '3090',
|
||||
url: [
|
||||
'https://www.amazon.co.uk/s?k=%2B%22RTX+3090%22+-3080+-GTX&i=computers&rh=n%3A430500031%2Cp_n_availability%3A419162031&s=relevancerank&dc&qid=1601675291',
|
||||
'https://www.amazon.co.uk/s?k=%2B%22RTX+3090%22+-3080+-GTX&i=computers&rh=n%3A430500031%2Cp_n_availability%3A419162031&s=relevancerank&dc&qid=1601675594&page=2'
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
name: 'amazon-uk'
|
||||
};
|
||||
@@ -0,0 +1,42 @@
|
||||
import {Store} from './store';
|
||||
import {getProductLinksBuilder} from './helpers/card';
|
||||
|
||||
export const Aria: Store = {
|
||||
labels: {
|
||||
inStock: {
|
||||
container: '#addQuantity',
|
||||
text: ['add to shopping basket']
|
||||
},
|
||||
outOfStock: {
|
||||
container: '.fBox',
|
||||
text: ['out of stock', 'there is currently no stock of this item']
|
||||
}
|
||||
},
|
||||
links: [
|
||||
{
|
||||
brand: 'test:brand',
|
||||
model: 'CARD',
|
||||
series: 'test:series',
|
||||
url: 'https://www.aria.co.uk/Products/Components/Graphics+Cards/NVIDIA+GeForce/GeForce+RTX+2060+Super/Gigabyte+NVIDIA+GeForce+RTX+2060+SUPER+8GB+WINDFORCE+OC+Turing+Graphics+Card+%2B+RTX+Bundle%21?productId=71541'
|
||||
}
|
||||
],
|
||||
linksBuilder: {
|
||||
builder: getProductLinksBuilder({
|
||||
productsSelector: '#productListingInner .listTable .listTableTr',
|
||||
sitePrefix: 'https://www.aria.co.uk',
|
||||
titleSelector: 'strong > a[href]'
|
||||
}),
|
||||
urls: [
|
||||
{
|
||||
series: '3080',
|
||||
url: 'https://www.aria.co.uk/Products/Components/Graphics+Cards/NVIDIA+GeForce/GeForce+RTX+3080'
|
||||
},
|
||||
{
|
||||
series: '3090',
|
||||
url: 'https://www.aria.co.uk/Products/Components/Graphics+Cards/NVIDIA+GeForce/GeForce+RTX+3090'
|
||||
}
|
||||
]
|
||||
},
|
||||
name: 'aria',
|
||||
waitUntil: 'domcontentloaded'
|
||||
};
|
||||
@@ -0,0 +1,45 @@
|
||||
import {Store} from './store';
|
||||
import {getProductLinksBuilder} from './helpers/card';
|
||||
|
||||
export const Box: Store = {
|
||||
labels: {
|
||||
inStock: {
|
||||
container: '#divBuyButton',
|
||||
text: ['add to basket']
|
||||
},
|
||||
outOfStock: {
|
||||
text: ['request stock alert', 'coming soon']
|
||||
}
|
||||
},
|
||||
links: [
|
||||
{
|
||||
brand: 'test:brand',
|
||||
model: 'CARD',
|
||||
series: 'test:series',
|
||||
url: 'https://www.box.co.uk/ASUS-TUF-GeForce-RTX-2060-6GB-Gaming-Gra_2669497.html'
|
||||
}
|
||||
],
|
||||
linksBuilder: {
|
||||
builder: getProductLinksBuilder({
|
||||
productsSelector: '.products-right .p-list',
|
||||
sitePrefix: 'https://www.box.co.uk',
|
||||
titleSelector: '.p-list-section > h3 > a[href]'
|
||||
}),
|
||||
urls: [
|
||||
{
|
||||
series: '3070',
|
||||
url: 'https://www.box.co.uk/rtx-3070-graphics-cards'
|
||||
},
|
||||
{
|
||||
series: '3080',
|
||||
url: 'https://www.box.co.uk/rtx-3080-graphics-cards'
|
||||
},
|
||||
{
|
||||
series: '3090',
|
||||
url: 'https://www.box.co.uk/rtx-3090-graphics-cards'
|
||||
}
|
||||
]
|
||||
},
|
||||
name: 'box',
|
||||
waitUntil: 'domcontentloaded'
|
||||
};
|
||||
@@ -0,0 +1,47 @@
|
||||
import {Store} from './store';
|
||||
import {getProductLinksBuilder} from './helpers/card';
|
||||
|
||||
export const Ccl: Store = {
|
||||
labels: {
|
||||
inStock: {
|
||||
container: '#pnlAddToBasket',
|
||||
text: ['add to basket']
|
||||
},
|
||||
outOfStock: {
|
||||
container: '#pnlSoldOut',
|
||||
text: ['sold out', 'coming soon']
|
||||
}
|
||||
},
|
||||
links: [
|
||||
{
|
||||
brand: 'test:brand',
|
||||
model: 'CARD',
|
||||
series: 'test:series',
|
||||
url: 'https://www.cclonline.com/product/296443/RTX-2060-SUPER-VENTUS-GP-OC/Graphics-Cards/MSI-GeForce-RTX-2060-SUPER-VENTUS-GP-OC-8GB-Overclocked-Graphics-Card/VGA5671/'
|
||||
}
|
||||
],
|
||||
linksBuilder: {
|
||||
builder: getProductLinksBuilder({
|
||||
productsSelector: '.productListingContainerOuter .productList',
|
||||
sitePrefix: 'https://www.cclonline.com',
|
||||
titleAttribute: 'title',
|
||||
titleSelector: '.productList_Detail a[title]'
|
||||
}),
|
||||
urls: [
|
||||
{
|
||||
series: '3070',
|
||||
url: 'https://www.cclonline.com/category/430/PC-Components/Graphics-Cards/GeForce-RTX-3070-Graphics-Cards/'
|
||||
},
|
||||
{
|
||||
series: '3080',
|
||||
url: 'https://www.cclonline.com/category/430/PC-Components/Graphics-Cards/GeForce-RTX-3080-Graphics-Cards/'
|
||||
},
|
||||
{
|
||||
series: '3090',
|
||||
url: 'https://www.cclonline.com/category/430/PC-Components/Graphics-Cards/GeForce-RTX-3090-Graphics-Cards/'
|
||||
}
|
||||
]
|
||||
},
|
||||
name: 'ccl',
|
||||
waitUntil: 'domcontentloaded'
|
||||
};
|
||||
@@ -0,0 +1,43 @@
|
||||
import {Store} from './store';
|
||||
import {getProductLinksBuilder} from './helpers/card';
|
||||
|
||||
export const Currys: Store = {
|
||||
labels: {
|
||||
inStock: {
|
||||
container: '#product-actions button',
|
||||
text: ['add to basket']
|
||||
},
|
||||
outOfStock: {
|
||||
container: '#product-actions .unavailable',
|
||||
text: ['not available for delivery']
|
||||
}
|
||||
},
|
||||
links: [
|
||||
{
|
||||
brand: 'test:brand',
|
||||
model: 'CARD',
|
||||
series: 'test:series',
|
||||
url: 'https://www.currys.co.uk/gbuk/computing-accessories/components-upgrades/graphics-cards/msi-geforce-rtx-2060-8-gb-super-ventus-gp-oc-graphics-card-10196803-pdt.html'
|
||||
}
|
||||
],
|
||||
linksBuilder: {
|
||||
builder: getProductLinksBuilder({
|
||||
productsSelector: '.resultList .product',
|
||||
sitePrefix: 'https://www.currys.co.uk',
|
||||
titleSelector: '.productTitle',
|
||||
urlSelector: 'a[href]'
|
||||
}),
|
||||
urls: [
|
||||
{
|
||||
series: '3080',
|
||||
url: 'https://www.currys.co.uk/gbuk/rtx-3080/components-upgrades/graphics-cards/324_3091_30343_xx_ba00013562-bv00313767/xx-criteria.html'
|
||||
},
|
||||
{
|
||||
series: '3090',
|
||||
url: 'https://www.currys.co.uk/gbuk/rtx-3090/components-upgrades/graphics-cards/324_3091_30343_xx_ba00013562-bv00313725/xx-criteria.html'
|
||||
}
|
||||
]
|
||||
},
|
||||
name: 'currys',
|
||||
waitUntil: 'domcontentloaded'
|
||||
};
|
||||
@@ -0,0 +1,47 @@
|
||||
import {Store} from './store';
|
||||
import {getProductLinksBuilder} from './helpers/card';
|
||||
|
||||
export const Ebuyer: Store = {
|
||||
labels: {
|
||||
inStock: {
|
||||
container: '.purchase-info',
|
||||
text: ['add to basket', 'in stock']
|
||||
},
|
||||
outOfStock: {
|
||||
container: '.purchase-info',
|
||||
text: ['coming soon', 'we are expecting this item on']
|
||||
}
|
||||
},
|
||||
links: [
|
||||
{
|
||||
brand: 'test:brand',
|
||||
model: 'CARD',
|
||||
series: 'test:series',
|
||||
url: 'https://www.ebuyer.com/874209-gigabyte-geforce-rtx-2060-windforce-6gb-oc-graphics-card-gv-n2060wf2oc-6gd-v2'
|
||||
}
|
||||
],
|
||||
linksBuilder: {
|
||||
builder: getProductLinksBuilder({
|
||||
productsSelector: '#list-view .listing-product',
|
||||
sitePrefix: 'https://www.ebuyer.com',
|
||||
titleSelector: '.listing-product-title',
|
||||
urlSelector: 'a[href]'
|
||||
}),
|
||||
urls: [
|
||||
{
|
||||
series: '3070',
|
||||
url: 'https://www.ebuyer.com/store/Components/cat/Graphics-Cards-Nvidia/subcat/GeForce-RTX-3070'
|
||||
},
|
||||
{
|
||||
series: '3080',
|
||||
url: 'https://www.ebuyer.com/store/Components/cat/Graphics-Cards-Nvidia/subcat/GeForce-RTX-3080'
|
||||
},
|
||||
{
|
||||
series: '3090',
|
||||
url: 'https://www.ebuyer.com/store/Components/cat/Graphics-Cards-Nvidia/subcat/GeForce-RTX-3090'
|
||||
}
|
||||
]
|
||||
},
|
||||
name: 'ebuyer',
|
||||
waitUntil: 'domcontentloaded'
|
||||
};
|
||||
@@ -1,13 +1,91 @@
|
||||
import {Link, Series} from '../store';
|
||||
import {logger} from '../../../logger';
|
||||
|
||||
export interface Card {
|
||||
brand: string;
|
||||
model: string;
|
||||
}
|
||||
|
||||
interface LinksBuilderOptions {
|
||||
productsSelector: string;
|
||||
sitePrefix: string;
|
||||
titleAttribute?: string;
|
||||
titleSelector: string;
|
||||
urlSelector?: string;
|
||||
}
|
||||
|
||||
const isPartialUrlRegExp = /^\/[^/]/i;
|
||||
|
||||
export function getProductLinksBuilder(options: LinksBuilderOptions) {
|
||||
/* eslint-disable unicorn/no-fn-reference-in-iterator */
|
||||
return (docElement: cheerio.Cheerio, series: Series): Link[] => {
|
||||
const productElements = docElement.find(options.productsSelector);
|
||||
const links: Link[] = [];
|
||||
for (let i = 0; i < productElements.length; i++) {
|
||||
const productElement = productElements.eq(i);
|
||||
const titleElement = productElement.find(options.titleSelector).first();
|
||||
let title: string;
|
||||
|
||||
if (options.titleAttribute) {
|
||||
title = titleElement.attr()?.[options.titleAttribute];
|
||||
} else {
|
||||
title = titleElement.text()?.replace(/\n/g, ' ').trim();
|
||||
}
|
||||
|
||||
if (!title) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let urlElement = titleElement;
|
||||
|
||||
if (options.urlSelector) {
|
||||
urlElement = urlElement.find(options.urlSelector).first();
|
||||
}
|
||||
|
||||
let url = urlElement.attr()?.href;
|
||||
|
||||
if (!url) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (isPartialUrlRegExp.exec(url)) {
|
||||
url = options.sitePrefix + url;
|
||||
}
|
||||
|
||||
const card = parseCard(title);
|
||||
|
||||
if (card) {
|
||||
links.push({
|
||||
brand: card.brand as any,
|
||||
model: card.model,
|
||||
series,
|
||||
url
|
||||
});
|
||||
} else {
|
||||
logger.error(`Failed to parse card: ${title}`);
|
||||
}
|
||||
}
|
||||
|
||||
return links;
|
||||
};
|
||||
/* eslint-enable unicorn/no-fn-reference-in-iterator */
|
||||
}
|
||||
|
||||
export function parseCard(name: string): Card | null {
|
||||
name = name.replace(/[^\w ]+/g, '').trim();
|
||||
name = name.replace(/\bgraphics card\b/gi, '').trim();
|
||||
name = name.replace(/\b\w+ fan\b/gi, '').trim();
|
||||
name = name.replace(/\s{2,}/g, ' ');
|
||||
name = name.replace(/\w+-\w+-[^ ]+/g, '');
|
||||
name = name.replace(/\([^(]*\)/g, '');
|
||||
name = name.replace(/, .+$/, '');
|
||||
name = name.replace(/ with .+$/, '');
|
||||
|
||||
// Account for incorrect titles, e.g. MSIGeforce
|
||||
name = name.replace(/geforce/i, '');
|
||||
|
||||
name = name.replace(/[^\w ]+/g, '');
|
||||
name = name.replace(/\bgraphics card\b/gi, '');
|
||||
name = name.replace(/\b(?<!founders) edition\b/gi, '');
|
||||
name = name.replace(/\b(series )?bundle\b/gi, '');
|
||||
name = name.replace(/\b\w+ fan\b/gi, '');
|
||||
name = name.replace(/\s{2,}/g, ' ').trim();
|
||||
|
||||
let model = name.split(' ');
|
||||
const brand = model.shift();
|
||||
@@ -16,6 +94,9 @@ export function parseCard(name: string): Card | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Split non spaced TitleCase words only after extracting brand
|
||||
model = model.join(' ').replace(/([A-Z][a-z]+)([A-Z][a-z]+)/g, '$1 $2').split(' ');
|
||||
|
||||
// Some vendors have oc at the beginning of the product name,
|
||||
// store whether the card contains the term "oc" and remove
|
||||
// it during filtering, then add it to the end of the name.
|
||||
@@ -28,8 +109,9 @@ export function parseCard(name: string): Card | null {
|
||||
return false;
|
||||
}
|
||||
|
||||
return !word.match(/^(nvidia|geforce|rtx|amp[ae]re|graphics|card|gpu|pci-?e(xpress)?|ray-?tracing|ray|tracing|core|boost)$/i) &&
|
||||
!word.match(/^(\d+(?:gb?|mhz)?|gb|mhz|g?ddr(\d+x?)?)$/i);
|
||||
return !word.match(/^(nvidia|geforce|ge|force|rtx|amp[ae]re|graphics|card|gpu|pci-?e(xpress)?|ray-?tracing|ray|tracing|core|boost|epicx)$/i) &&
|
||||
!word.match(/^(\d+(?:gb?|mhz)?|gb|mhz|g?ddr(\d+x?)?)$/i) &&
|
||||
!word.match(/^(display ?port|hdmi|vga)$/i);
|
||||
});
|
||||
/* eslint-enable @typescript-eslint/prefer-regexp-exec */
|
||||
|
||||
|
||||
@@ -3,22 +3,32 @@ import {Amazon} from './amazon';
|
||||
import {AmazonCa} from './amazon-ca';
|
||||
import {AmazonDe} from './amazon-de';
|
||||
import {AmazonNl} from './amazon-nl';
|
||||
import {AmazonUk} from './amazon-uk';
|
||||
import {Aria} from './aria';
|
||||
import {Asus} from './asus';
|
||||
import {AsusDe} from './asus-de';
|
||||
import {BAndH} from './bandh';
|
||||
import {BestBuy} from './bestbuy';
|
||||
import {BestBuyCa} from './bestbuy-ca';
|
||||
import {Box} from './box';
|
||||
import {Ccl} from './ccl';
|
||||
import {Currys} from './currys';
|
||||
import {Ebuyer} from './ebuyer';
|
||||
import {Evga} from './evga';
|
||||
import {EvgaEu} from './evga-eu';
|
||||
import {Gamestop} from './gamestop';
|
||||
import {MicroCenter} from './microcenter';
|
||||
import {Newegg} from './newegg';
|
||||
import {NeweggCa} from './newegg-ca';
|
||||
import {Novatech} from './novatech';
|
||||
import {Nvidia} from './nvidia';
|
||||
import {NvidiaApi} from './nvidia-api';
|
||||
import {OfficeDepot} from './officedepot';
|
||||
import {Overclockers} from './overclockers';
|
||||
import {Pny} from './pny';
|
||||
import {Scan} from './scan';
|
||||
import {Store} from './store';
|
||||
import {Very} from './very';
|
||||
import {Zotac} from './zotac';
|
||||
import {config} from '../../config';
|
||||
import {logger} from '../../logger';
|
||||
@@ -29,21 +39,31 @@ const masterList = new Map([
|
||||
[AmazonCa.name, AmazonCa],
|
||||
[AmazonDe.name, AmazonDe],
|
||||
[AmazonNl.name, AmazonNl],
|
||||
[AmazonUk.name, AmazonUk],
|
||||
[Aria.name, Aria],
|
||||
[Asus.name, Asus],
|
||||
[AsusDe.name, AsusDe],
|
||||
[BAndH.name, BAndH],
|
||||
[BestBuy.name, BestBuy],
|
||||
[BestBuyCa.name, BestBuyCa],
|
||||
[Box.name, Box],
|
||||
[Ccl.name, Ccl],
|
||||
[Currys.name, Currys],
|
||||
[Ebuyer.name, Ebuyer],
|
||||
[Evga.name, Evga],
|
||||
[EvgaEu.name, EvgaEu],
|
||||
[Gamestop.name, Gamestop],
|
||||
[MicroCenter.name, MicroCenter],
|
||||
[Newegg.name, Newegg],
|
||||
[NeweggCa.name, NeweggCa],
|
||||
[Novatech.name, Novatech],
|
||||
[Nvidia.name, Nvidia],
|
||||
[NvidiaApi.name, NvidiaApi],
|
||||
[OfficeDepot.name, OfficeDepot],
|
||||
[Overclockers.name, Overclockers],
|
||||
[Pny.name, Pny],
|
||||
[Scan.name, Scan],
|
||||
[Very.name, Very],
|
||||
[Zotac.name, Zotac]
|
||||
]);
|
||||
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
import {Store} from './store';
|
||||
import {getProductLinksBuilder} from './helpers/card';
|
||||
|
||||
export const Novatech: Store = {
|
||||
labels: {
|
||||
inStock: {
|
||||
container: '.newspec-specprice',
|
||||
text: ['add to basket']
|
||||
},
|
||||
outOfStock: {
|
||||
container: '.newspec-pricesection',
|
||||
text: [
|
||||
'very short supply, no confirmed date',
|
||||
'this product is only available to buy when in stock',
|
||||
'ordered upon request',
|
||||
'price to be confirmed'
|
||||
]
|
||||
}
|
||||
},
|
||||
links: [
|
||||
{
|
||||
brand: 'test:brand',
|
||||
model: 'CARD',
|
||||
series: 'test:series',
|
||||
url: 'https://www.novatech.co.uk/products/gigabyte-geforce-rtx-2060-oc-v2-6g-graphics-card/gv-n2060oc-6gdv2.html'
|
||||
}
|
||||
],
|
||||
linksBuilder: {
|
||||
builder: getProductLinksBuilder({
|
||||
productsSelector: '.seo-container .search-box-results',
|
||||
sitePrefix: 'https://www.novatech.co.uk',
|
||||
titleSelector: '.search-box-title',
|
||||
urlSelector: 'a[href]'
|
||||
}),
|
||||
urls: [
|
||||
{
|
||||
series: '3080',
|
||||
url: 'https://www.novatech.co.uk/products/components/nvidiageforcegraphicscards/nvidiartxseries/nvidiartx3080/?i=200'
|
||||
},
|
||||
{
|
||||
series: '3090',
|
||||
url: 'https://www.novatech.co.uk/products/components/nvidiageforcegraphicscards/nvidiartxseries/nvidiartx3090/?i=200'
|
||||
}
|
||||
]
|
||||
},
|
||||
name: 'novatech',
|
||||
waitUntil: 'domcontentloaded'
|
||||
};
|
||||
@@ -0,0 +1,51 @@
|
||||
import {Store} from './store';
|
||||
import {getProductLinksBuilder} from './helpers/card';
|
||||
|
||||
export const Overclockers: Store = {
|
||||
labels: {
|
||||
inStock: {
|
||||
container: '#detailbox',
|
||||
text: ['add to basket', 'in stock']
|
||||
},
|
||||
outOfStock: {
|
||||
container: '#detailbox',
|
||||
text: ['out of stock', 'pre order', 'bought to order']
|
||||
}
|
||||
},
|
||||
links: [
|
||||
{
|
||||
brand: 'test:brand',
|
||||
model: 'CARD',
|
||||
series: 'test:series',
|
||||
url: 'https://www.overclockers.co.uk/gigabyte-geforce-rtx-2060-oc-rev2-6144mb-gddr6-pci-express-graphics-card-gx-1bj-gi.html'
|
||||
}
|
||||
],
|
||||
linksBuilder: {
|
||||
builder: getProductLinksBuilder({
|
||||
productsSelector: '.ck_listing .artbox',
|
||||
sitePrefix: 'https://www.overclockers.co.uk',
|
||||
titleAttribute: 'data-description',
|
||||
titleSelector: 'a[href].producttitles'
|
||||
}),
|
||||
urls: [
|
||||
{
|
||||
series: '3070',
|
||||
url: 'https://www.overclockers.co.uk/pc-components/graphics-cards/nvidia/geforce-rtx-3070'
|
||||
},
|
||||
{
|
||||
series: '3080',
|
||||
// Need to add support to detect pagination so this can be dynamically detected
|
||||
url: [
|
||||
'https://www.overclockers.co.uk/pc-components/graphics-cards/nvidia/geforce-rtx-3080',
|
||||
'https://www.overclockers.co.uk/pc-components/graphics-cards/nvidia/geforce-rtx-3080?p=2'
|
||||
]
|
||||
},
|
||||
{
|
||||
series: '3090',
|
||||
url: 'https://www.overclockers.co.uk/pc-components/graphics-cards/nvidia/geforce-rtx-3090'
|
||||
}
|
||||
]
|
||||
},
|
||||
name: 'overclockers',
|
||||
waitUntil: 'domcontentloaded'
|
||||
};
|
||||
@@ -0,0 +1,52 @@
|
||||
import {Store} from './store';
|
||||
import {getProductLinksBuilder} from './helpers/card';
|
||||
|
||||
export const Scan: Store = {
|
||||
disableAdBlocker: true,
|
||||
labels: {
|
||||
captcha: [{
|
||||
container: '#challenge-form',
|
||||
text: ['hcaptcha_submit']
|
||||
}],
|
||||
inStock: {
|
||||
container: '.buyPanel .priceAvailability',
|
||||
text: ['add to basket', 'in stock']
|
||||
},
|
||||
outOfStock: {
|
||||
container: '.buyPanel .priceAvailability',
|
||||
text: ['pre order']
|
||||
}
|
||||
},
|
||||
links: [
|
||||
{
|
||||
brand: 'test:brand',
|
||||
model: 'CARD',
|
||||
series: 'test:series',
|
||||
url: 'https://www.scan.co.uk/products/msi-geforce-rtx-2060-ventus-xs-oc-6gb-gddr6-vr-ready-graphics-card-1920-core-1710mhz-boost'
|
||||
}
|
||||
],
|
||||
linksBuilder: {
|
||||
builder: getProductLinksBuilder({
|
||||
productsSelector: 'ul.productColumns li.product',
|
||||
sitePrefix: 'https://www.scan.co.uk',
|
||||
titleSelector: '.details .description',
|
||||
urlSelector: 'a[href]'
|
||||
}),
|
||||
urls: [
|
||||
{
|
||||
series: '3070',
|
||||
url: 'https://www.scan.co.uk/shop/computer-hardware/gpu-nvidia/nvidia-geforce-rtx-3070-graphics-cards'
|
||||
},
|
||||
{
|
||||
series: '3080',
|
||||
url: 'https://www.scan.co.uk/shop/computer-hardware/gpu-nvidia/nvidia-geforce-rtx-3080-graphics-cards'
|
||||
},
|
||||
{
|
||||
series: '3090',
|
||||
url: 'https://www.scan.co.uk/shop/computer-hardware/gpu-nvidia/nvidia-geforce-rtx-3090-graphics-cards'
|
||||
}
|
||||
]
|
||||
},
|
||||
name: 'scan',
|
||||
waitUntil: 'domcontentloaded'
|
||||
};
|
||||
@@ -46,7 +46,8 @@ export type Store = {
|
||||
links: Link[];
|
||||
linksBuilder?: {
|
||||
builder: (docElement: cheerio.Cheerio, series: Series) => Link[];
|
||||
urls: Array<{series: Series; url: string}>;
|
||||
ttl?: number;
|
||||
urls: Array<{series: Series; url: string | string[]}>;
|
||||
};
|
||||
labels: Labels;
|
||||
name: string;
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
import {Link, Store} from './store';
|
||||
import {logger} from '../../logger';
|
||||
import {parseCard} from './helpers/card';
|
||||
|
||||
export const Very: Store = {
|
||||
labels: {
|
||||
inStock: {
|
||||
container: '.stockMessaging .indicator',
|
||||
text: ['available', 'low stock']
|
||||
},
|
||||
outOfStock: {
|
||||
container: '.stockMessaging .indicator',
|
||||
text: ['pre-order']
|
||||
}
|
||||
},
|
||||
links: [
|
||||
{
|
||||
brand: 'test:brand',
|
||||
model: 'CARD',
|
||||
series: 'test:series',
|
||||
url: 'https://www.very.co.uk/msi-geforce-rtx-2060-super-ventus-gp-oc/1600463772.prd'
|
||||
}
|
||||
],
|
||||
linksBuilder: {
|
||||
builder: (docElement, series) => {
|
||||
const productElements = docElement.find('.productList .product');
|
||||
const links: Link[] = [];
|
||||
for (let i = 0; i < productElements.length; i++) {
|
||||
const productElement = productElements.eq(i);
|
||||
const titleElement = productElement.find('.productTitle').first();
|
||||
const title = titleElement.text()?.replace(/\n/g, ' ').trim();
|
||||
|
||||
if (!title || ['RTX', series]
|
||||
.map(x => title.toLowerCase().includes(x.toLowerCase()))
|
||||
.filter(x => !x).length > 0
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const url = titleElement.attr()?.href;
|
||||
|
||||
if (!url) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const card = parseCard(title);
|
||||
|
||||
if (card) {
|
||||
links.push({
|
||||
brand: card.brand as any,
|
||||
model: card.model,
|
||||
series,
|
||||
url
|
||||
});
|
||||
} else {
|
||||
logger.error(`Failed to parse card: ${title}`);
|
||||
}
|
||||
}
|
||||
|
||||
return links;
|
||||
},
|
||||
ttl: 300000,
|
||||
urls: [
|
||||
{
|
||||
series: '3070',
|
||||
url: 'https://www.very.co.uk/electricals/pc-components/graphics-cards/e/b/118786.end?sort=newin,0&numProducts=100'
|
||||
},
|
||||
{
|
||||
series: '3080',
|
||||
url: 'https://www.very.co.uk/electricals/pc-components/graphics-cards/e/b/118786.end?sort=newin,0&numProducts=100'
|
||||
},
|
||||
{
|
||||
series: '3090',
|
||||
url: 'https://www.very.co.uk/electricals/pc-components/graphics-cards/e/b/118786.end?sort=newin,0&numProducts=100'
|
||||
}
|
||||
]
|
||||
},
|
||||
name: 'very'
|
||||
};
|
||||
Reference in New Issue
Block a user