opelly/streetmerchant
1
0
Fork 0
mirror of https://github.com/opelly27/streetmerchant.git synced 2026-05-20 06:27:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(scraping): change lookup impl, add randomize sleep (#110)

Browse Source 
Co-authored-by: Jef LeCompte <jeffreylec@gmail.com>
This commit is contained in:
Mark Dietzer
2020-09-20 10:52:37 -07:00
committed by GitHub
parent 2e5d13bda5
commit dc0f710674
5 changed files with 26 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/config.ts
+3 -2
View File
@@ -6,7 +6,8 @@ config({path: resolve(__dirname, '../.env')});
const browser = {
isHeadless: process.env.HEADLESS ? process.env.HEADLESS === 'true' : true,
open: process.env.OPEN_BROWSER === 'true',
rateLimitTimeout: process.env.RATE_LIMIT_TIMEOUT ? Number(process.env.RATE_LIMIT_TIMEOUT) : 5000
minSleep: Number(process.env.PAGE_SLEEP_MIN ?? 5000),
maxSleep: Number(process.env.PAGE_SLEEP_MAX ?? 10000)
};
const logLevel = process.env.LOG_LEVEL ?? 'info';
@@ -53,7 +54,7 @@ const page = {
capture: process.env.SCREENSHOT === 'true',
width: 1920,
height: 1080,
navigationTimeout: Number(process.env.PAGE_TIMEOUT) ?? 30000,
navigationTimeout: Number(process.env.PAGE_TIMEOUT ?? 30000),
userAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'
};