opelly/streetmerchant
1
0
Fork 0
mirror of https://github.com/opelly27/streetmerchant.git synced 2026-05-20 06:27:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(scraping): change lookup impl, add randomize sleep (#110)

Browse Source 
Co-authored-by: Jef LeCompte <jeffreylec@gmail.com>
This commit is contained in:
Mark Dietzer
2020-09-20 10:52:37 -07:00
committed by GitHub
parent 2e5d13bda5
commit dc0f710674
5 changed files with 26 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/config.ts
+3 -2
View File
@@ -6,7 +6,8 @@ config({path: resolve(__dirname, '../.env')});
const browser = {
isHeadless: process.env.HEADLESS ? process.env.HEADLESS === 'true' : true,
open: process.env.OPEN_BROWSER === 'true',
rateLimitTimeout: process.env.RATE_LIMIT_TIMEOUT ? Number(process.env.RATE_LIMIT_TIMEOUT) : 5000
minSleep: Number(process.env.PAGE_SLEEP_MIN ?? 5000),
maxSleep: Number(process.env.PAGE_SLEEP_MAX ?? 10000)
};
const logLevel = process.env.LOG_LEVEL ?? 'info';
@@ -53,7 +54,7 @@ const page = {
capture: process.env.SCREENSHOT === 'true',
width: 1920,
height: 1080,
navigationTimeout: Number(process.env.PAGE_TIMEOUT) ?? 30000,
navigationTimeout: Number(process.env.PAGE_TIMEOUT ?? 30000),
userAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'
};
src/index.ts
+19 -24
View File
@@ -5,11 +5,28 @@ import {Config} from './config';
import {Store, Stores} from './store/model';
import {Logger} from './logger';
import {lookup} from './store';
import async from 'async';
import {Browser} from 'puppeteer';
puppeteer.use(stealthPlugin());
puppeteer.use(adblockerPlugin({blockTrackers: true}));
function getSleepTime() {
return Config.browser.minSleep + (Math.random() * (Config.browser.maxSleep - Config.browser.minSleep));
}
async function tryLookupAndLoop(browser: Browser, store: Store) {
Logger.debug(`[${store.name}] Starting lookup...`);
try {
await lookup(browser, store);
} catch (error) {
Logger.error(error);
}
const sleepTime = getSleepTime();
Logger.debug(`[${store.name}] Lookup done, next one in ${sleepTime} ms`);
setTimeout(tryLookupAndLoop, sleepTime, browser, store);
}
/**
* Starts the bot.
*/
@@ -22,32 +39,10 @@ async function main() {
}
});
const q = async.queue<Store>(async (store: Store, cb) => {
setTimeout(async () => {
try {
Logger.debug(`↗ scraping initialized - ${store.name}`);
await lookup(browser, store);
} catch (error) {
// Ignoring errors; more than likely due to rate limits
Logger.error(error);
} finally {
cb();
q.push(store);
}
}, Config.browser.rateLimitTimeout);
}, Stores.length);
for (const store of Stores) {
Logger.debug(store.links);
q.push(store);
if (Stores.length === 1) {
q.push(store);
} // Keep from completely draining
setTimeout(tryLookupAndLoop, getSleepTime(), browser, store);
}
await q.drain();
await browser.close();
}
/**