feat(api): add rudimentary web control panel (#183)

.env-example

@@ -70,3 +70,4 @@ TWITTER_CONSUMER_KEY=
 TWITTER_CONSUMER_SECRET=
 TWITTER_TWEET_TAGS=
 USER_AGENT=
+WEB_PORT=

README.md

@@ -149,6 +149,7 @@ Here is a list of variables that you can use to customize your newly copied `.en
 | `TWITCH_REFRESH_TOKEN` | Twitch refresh token | |
 | `TWITCH_CHANNEL` | Twitch channel | |
 | `USER_AGENT` | Custom User-Agents headers for HTTP requests | Newline separated, e.g.: `USER_AGENT_STRING1 \n USER_AGENT_STRING2` | | Default: `Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36` |
+| `WEB_PORT` | Starts a webserver to be able to control the bot while it is running; optional | Default: disabled |
 
 > :point_right: If you have multi-factor authentication (MFA), you will need to create an [app password](https://myaccount.google.com/apppasswords) and use this instead of your Gmail password.
 

src/config.ts

@@ -271,3 +271,10 @@ export const config = {
 	proxy,
 	store
 };
+
+export function setConfig(newConfig: any) {
+	const writeConfig = config as any;
+	for (const key of Object.keys(newConfig)) {
+		writeConfig[key] = newConfig[key];
+	}
+}

src/index.ts

@@ -1,4 +1,5 @@
-import {Stores} from './store/model';
+import {startAPIServer, stopAPIServer} from './web';
+import {Browser} from 'puppeteer';
 import {adBlocker} from './adblocker';
 import {config} from './config';
 import {getSleepTime} from './util';
@@ -6,6 +7,7 @@ import {logger} from './logger';
 import puppeteer from 'puppeteer-extra';
 import resourceBlock from 'puppeteer-extra-plugin-block-resources';
 import stealthPlugin from 'puppeteer-extra-plugin-stealth';
+import {storeList} from './store/model';
 import {tryLookupAndLoop} from './store';
 
 puppeteer.use(stealthPlugin());
@@ -17,15 +19,12 @@ if (config.browser.lowBandwidth) {
 	puppeteer.use(adBlocker);
 }
 
+let browser: Browser | undefined;
+
 /**
  * Starts the bot.
  */
 async function main() {
-	if (Stores.length === 0) {
-		logger.error('✖ no stores selected', Stores);
-		return;
-	}
-
 	const args: string[] = [];
 
 	// Skip Chromium Linux Sandbox
@@ -45,7 +44,9 @@ async function main() {
 		args.push(`--proxy-server=http://${config.proxy.address}:${config.proxy.port}`);
 	}
 
-	const browser = await puppeteer.launch({
+	await stop();
+
+	browser = await puppeteer.launch({
 		args,
 		defaultViewport: {
 			height: config.page.height,
@@ -54,7 +55,7 @@ async function main() {
 		headless: config.browser.isHeadless
 	});
 
-	for (const store of Stores) {
+	for (const store of storeList.values()) {
 		logger.debug('store links', {meta: {links: store.links}});
 		if (store.setupAction !== undefined) {
 			store.setupAction(browser);
@@ -62,14 +63,41 @@ async function main() {
 
 		setTimeout(tryLookupAndLoop, getSleepTime(store), browser, store);
 	}
+
+	await startAPIServer();
+}
+
+async function stop() {
+	await stopAPIServer();
+
+	if (browser) {
+		// Use temporary swap variable to avoid any race condition
+		const browserTemporary = browser;
+		browser = undefined;
+		await browserTemporary.close();
+	}
+}
+
+async function stopAndExit() {
+	await stop();
+	// eslint-disable-next-line unicorn/no-process-exit
+	process.exit(0);
 }
 
 /**
  * Will continually run until user interferes.
  */
-try {
-	void main();
-} catch (error) {
-	logger.error('✖ something bad happened, resetting nvidia-snatcher', error);
-	void main();
+async function loopMain() {
+	try {
+		await main();
+	} catch (error) {
+		logger.error('✖ something bad happened, resetting nvidia-snatcher in 5 seconds', error);
+		setTimeout(loopMain, 5000);
+	}
 }
+
+void loopMain();
+
+process.on('SIGINT', stopAndExit);
+process.on('SIGQUIT', stopAndExit);
+process.on('SIGTERM', stopAndExit);

src/store/lookup.ts

@@ -24,6 +24,10 @@ const linkBuilderLastRunTimes: Record<string, number> = {};
  * @param store Vendor of graphics cards.
  */
 async function lookup(browser: Browser, store: Store) {
+	if (config.store.stores.length > 0 && !config.store.stores.find(foundStore => foundStore.name === store.name)) {
+		return;
+	}
+
 	/* eslint-disable no-await-in-loop */
 	for (const link of store.links) {
 		if (!filterStoreLink(link)) {
@@ -192,6 +196,11 @@ async function lookupCardInStock(store: Store, page: Page, link: Link) {
 }
 
 export async function tryLookupAndLoop(browser: Browser, store: Store) {
+	if (!browser.isConnected()) {
+		logger.debug(`[${store.name}] Ending this loop as browser is disposed...`);
+		return;
+	}
+
 	if (store.linksBuilder) {
 		const lastRunTime = linkBuilderLastRunTimes[store.name] ?? -1;
 		const ttl = store.linksBuilder.ttl ?? Number.MAX_SAFE_INTEGER;

src/store/model/index.ts

@@ -42,13 +42,10 @@ import {ProshopDE} from './proshop-de';
 import {ProshopDK} from './proshop-dk';
 import {Saturn} from './saturn';
 import {Scan} from './scan';
-import {Store} from './store';
 import {Very} from './very';
 import {Zotac} from './zotac';
-import {config} from '../../config';
-import {logger} from '../../logger';
 
-const masterList = new Map([
+export const storeList = new Map([
 	[Adorama.name, Adorama],
 	[Alternate.name, Alternate],
 	[AlternateNL.name, AlternateNL],
@@ -97,33 +94,27 @@ const masterList = new Map([
 	[Zotac.name, Zotac]
 ]);
 
-const list = new Map();
-
-for (const storeData of config.store.stores) {
-	if (masterList.has(storeData.name)) {
-		list.set(storeData.name, {...masterList.get(storeData.name), ...storeData});
-	} else {
-		const logString = `No store named ${storeData.name}, skipping.`;
-		logger.warn(logString);
+const brands = new Set();
+const series = new Set();
+const models = new Set();
+for (const store of storeList.values()) {
+	for (const link of store.links) {
+		brands.add(link.brand);
+		series.add(link.series);
+		models.add(link.model);
 	}
 }
 
-logger.info(`ℹ selected stores: ${Array.from(list.keys()).join(', ')}`);
-
-if (config.store.showOnlyBrands.length > 0) {
-	logger.info(`ℹ selected brands: ${config.store.showOnlyBrands.join(', ')}`);
+export function getAllBrands() {
+	return Array.from(brands);
 }
 
-if (config.store.showOnlyModels.length > 0) {
-	logger.info(`ℹ selected models: ${config.store.showOnlyModels.map(entry => {
-		return entry.series ? entry.name + ' (' + entry.series + ')' : entry.name;
-	}).join(', ')}`);
+export function getAllSeries() {
+	return Array.from(series);
 }
 
-if (config.store.showOnlySeries.length > 0) {
-	logger.info(`ℹ selected series: ${config.store.showOnlySeries.join(', ')}`);
+export function getAllModels() {
+	return Array.from(models);
 }
 
-export const Stores = Array.from(list.values()) as Store[];
-
 export * from './store';

src/web/index.ts

@@ -0,0 +1,169 @@
+import {IncomingMessage, Server, ServerResponse, createServer} from 'http';
+import {config, setConfig} from '../config';
+import {createReadStream, readdir} from 'fs';
+import {getAllBrands, getAllModels, getAllSeries, storeList} from '../store/model';
+import {join, normalize} from 'path';
+
+const approot = join(__dirname, '../../');
+const webroot = join(approot, './web');
+
+const contentTypeMap: { [key: string]: string } = {
+	css: 'text/css',
+	htm: 'text/html',
+	html: 'text/html',
+	jpeg: 'image/jpeg',
+	jpg: 'image/jpeg',
+	js: 'application/javascript',
+	json: 'application/json',
+	png: 'image/png',
+	txt: 'text/plain'
+};
+
+function sendFile(response: ServerResponse, path: string, relativeTo: string = webroot) {
+	path = normalize(`./${path}`);
+
+	const fsPath = join(relativeTo, path);
+	try {
+		const stream = createReadStream(fsPath);
+		stream.on('error', error => {
+			sendError(response, error.message);
+		});
+
+		const pathSplit = path.split('.');
+		const ext = pathSplit[pathSplit.length - 1].toLowerCase();
+
+		response.setHeader('Content-Type', contentTypeMap[ext] ?? contentTypeMap.txt);
+
+		stream.on('end', () => response.end());
+		stream.pipe(response);
+	} catch (error) {
+		sendError(response, error);
+	}
+}
+
+function sendError(response: ServerResponse, data: string, statusCode = 500) {
+	response.statusCode = statusCode;
+	response.setHeader('Content-Type', contentTypeMap.txt);
+	response.write(data);
+	response.end();
+}
+
+function sendJSON(response: ServerResponse, data: any) {
+	response.setHeader('Content-Type', contentTypeMap.json);
+	response.write(JSON.stringify(data));
+	response.end();
+}
+
+function sendConfig(response: ServerResponse) {
+	sendJSON(response, config);
+}
+
+function handleAPI(request: IncomingMessage, response: ServerResponse, urlComponents: string[]) {
+	if (urlComponents.length < 2) {
+		sendError(response, 'No API route specified', 400);
+		return;
+	}
+
+	switch (urlComponents[1]) {
+		case 'config':
+			if (request.method === 'PUT') {
+				const data: string[] = [];
+				request.on('data', chunk => {
+					data.push(chunk);
+				});
+				request.on('end', () => {
+					// We ignore errors, client just sent wrong data...
+					try {
+						setConfig(JSON.parse(data.join('')));
+					} catch { }
+
+					sendConfig(response);
+				});
+				return;
+			}
+
+			sendConfig(response);
+			return;
+		case 'stores':
+			sendJSON(response, Array.from(storeList.keys()));
+			return;
+		case 'brands':
+			sendJSON(response, getAllBrands());
+			return;
+		case 'series':
+			sendJSON(response, getAllSeries());
+			return;
+		case 'models':
+			sendJSON(response, getAllModels());
+			return;
+		case 'screenshots':
+			if (urlComponents.length >= 3) {
+				const timeStamp = urlComponents[2];
+				if (/\D/.test(timeStamp)) {
+					sendError(response, 'Invalid screenshot timestamp', 400);
+					return;
+				}
+
+				sendFile(response, `../success-${timeStamp}.png`);
+				return;
+			}
+
+			readdir(approot, (error, files) => {
+				if (error) {
+					sendError(response, error.message);
+					return;
+				}
+
+				const result = [];
+				for (const file of files) {
+					const match = /^success-(\d+)\.png$/.exec(file);
+					if (!match) {
+						continue;
+					}
+
+					result.push(match[1]);
+				}
+
+				sendJSON(response, result);
+			});
+			return;
+		default:
+			sendError(response, 'No API route found for path', 400);
+	}
+}
+
+function requestListener(request: IncomingMessage, response: ServerResponse) {
+	const url = request.url!;
+	const urlComponents = url.slice(1).split('/'); // Remove the leading /
+
+	switch (urlComponents[0]) {
+		case 'api':
+			handleAPI(request, response, urlComponents);
+			break;
+		default:
+			sendFile(response, url === '/' ? '/index.html' : url);
+			break;
+	}
+}
+
+let server: Server | undefined;
+
+export async function startAPIServer() {
+	await stopAPIServer();
+	if (process.env.WEB_PORT) {
+		server = createServer(requestListener);
+		server.listen(Number(process.env.WEB_PORT));
+	}
+}
+
+export async function stopAPIServer() {
+	return new Promise(resolve => {
+		if (server) {
+			server.close(resolve);
+			server = undefined;
+			return;
+		}
+
+		resolve();
+	});
+}

tsconfig.json

@@ -1,8 +1,8 @@
 {
   "compilerOptions": {
-    "target": "es5",
+    "target": "es2019",
     "module": "commonjs",
-    "lib": ["es6", "dom"],
+    "lib": ["dom", "es2019", "es2020.bigint", "es2020.string", "es2020.symbol.wellknown"],
     "allowJs": true,
     "outDir": "build",
     "rootDir": "src",
@@ -12,6 +12,6 @@
     "resolveJsonModule": true,
     "skipLibCheck": true,
     "forceConsistentCasingInFileNames": true,
-	"sourceMap": true
+    "sourceMap": true
   }
 }

web/index.html

@@ -0,0 +1,149 @@
+<!DOCTYPE html>
+<html>
+	<head>
+		<title>nvidia-snatcher control</title>
+		<script type="text/javascript">
+			let config;
+			let brands;
+			let stores;
+			let series;
+			let models;
+
+			function renderList(id, elements, selectionArray) {
+				const list = document.getElementById(id);
+				list.innerHTML = '';
+				for (const element of elements) {
+					const name = `${id}_${element}`;
+					const htmlElement = document.createElement('input');
+					htmlElement.type = 'checkbox';
+					htmlElement.value = element;
+					htmlElement.innerHTML = element;
+					htmlElement.name = name;
+					htmlElement.id = name;
+					if (selectionArray.length === 0 || selectionArray.includes(element)) {
+						htmlElement.checked = true;
+					}
+					list.appendChild(htmlElement);
+					const htmlLabelElement = document.createElement('label');
+					htmlLabelElement.for = name;
+					htmlLabelElement.innerText = element;
+					list.appendChild(htmlLabelElement);
+					list.appendChild(document.createElement('br'));
+				}
+			}
+
+			function listToArray(id, selectionArray) {
+				const list = document.getElementById(id);
+				const resArray = [];
+				let allSelected = true;
+				for (const htmlElement of list.childNodes) {
+					if (htmlElement.checked) {
+						resArray.push(htmlElement.value);
+					} else {
+						allSelected = false;
+					}
+				}
+
+				if (allSelected) {
+					return [];
+				}
+				return resArray;
+			}
+
+			async function onReceiveConfig(resp) {
+				config = await resp.json();
+
+				renderList('storeList', stores, config.store.stores);
+				renderList('brandList', brands, config.store.showOnlyBrands);
+				renderList('seriesList', series, config.store.showOnlySeries);
+				renderList('modelList', models, config.store.showOnlyModels);
+			}
+
+			async function setConfig() {
+				if (!config) {
+					throw new Error('Config not loaded yet');
+				}
+
+				config.store.stores = listToArray('storeList');
+				config.store.showOnlyBrands = listToArray('brandList');
+				config.store.showOnlySeries = listToArray('seriesList');
+				config.store.showOnlyModels = listToArray('modelList');
+
+				const resp = await fetch('/api/config', {
+					method: 'PUT',
+					headers: {
+						'Content-Type': 'application/json'
+					},
+					body: JSON.stringify(config)
+				});
+
+				await onReceiveConfig(resp);
+
+				alert('Saved!');
+			}
+
+			async function loadScreenshots() {
+				const respScreenshots = await fetch('/api/screenshots');
+				const screenshots = await respScreenshots.json();
+
+				const screenshotContainer = document.getElementById('screenshots');
+				screenshotContainer.innerHTML = '';
+
+				for (const screenshot of screenshots) {
+					const htmlElement = document.createElement('a');
+					htmlElement.href = `/api/screenshots/${screenshot}`;
+					htmlElement.target = '_blank';
+					htmlElement.innerText = screenshot;
+					screenshotContainer.appendChild(htmlElement);
+					screenshotContainer.appendChild(document.createElement('br'));
+				}
+			}
+
+			async function loadConfig() {
+				const resp = await fetch('/api/config');
+				await onReceiveConfig(resp);
+			}
+
+			async function loadInitial() {
+				const respStores = await fetch('/api/stores');
+				stores = await respStores.json();
+
+				const respBrands = await fetch('/api/brands');
+				brands = await respBrands.json();
+
+				const respSeries = await fetch('/api/series');
+				series = await respSeries.json();
+
+				const respModels = await fetch('/api/models');
+				models = await respModels.json();
+
+				await loadConfig();
+
+				await loadScreenshots();
+			}
+		</script>
+	</head>
+	<body onload="loadInitial();">
+		<table>
+			<tr>
+				<th>Stores</th>
+				<th>Brands</th>
+				<th>Series</th>
+				<th>Models</th>
+			</tr>
+			<tr>
+				<td valign="top" id="storeList"></td>
+				<td valign="top" id="brandList"></td>
+				<td valign="top" id="seriesList"></td>
+				<td valign="top" id="modelList"></td>
+			</tr>
+		</table>
+
+		<br />
+		<input type="button" onclick="setConfig();" value="Save" />
+
+		<br /><br /><br /><br />
+		<b>Screenshots (<a href="javascript:loadScreenshots();">Refresh</a>)</b>
+		<div id="screenshots"></div>
+	</body>
+</html>