opelly/streetmerchant
1
0
Fork 0
mirror of https://github.com/opelly27/streetmerchant.git synced 2026-05-20 05:17:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: page timeout (#22)

Browse Source 
Co-authored-by: Jef LeCompte <jeffreylec@gmail.com>
This commit is contained in:
Ion Caza
2020-09-18 19:04:29 -04:00
committed by GitHub
parent 7d7bd18b4d
commit 643045c7e0
4 changed files with 5 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -3,3 +3,4 @@ build/
node_modules/ node_modules/
.env .env
success-*.png
README.md
+1
View File
@@ -67,6 +67,7 @@ First, you're going to need to copy the `.env.example` to `.env`. The current op
| `EMAIL_USERNAME` | Gmail address; e.g. `jensen.robbed.us@gmail.com` | | `EMAIL_USERNAME` | Gmail address; e.g. `jensen.robbed.us@gmail.com` |
| `EMAIL_PASSWORD` | Gmail password; see below if you have MFA | | `EMAIL_PASSWORD` | Gmail password; see below if you have MFA |
| `STORES` | List of [stores](#Supported-stores) you want to be scraped; optional, default: `nvidia` | | `STORES` | List of [stores](#Supported-stores) you want to be scraped; optional, default: `nvidia` |
| `PAGE_TIMEOUT` | Navigation Timeout in milliseconds (0 for infinite, 60000 for 1 minute); optional, default: `0` |
> :point_right: If you have multi-factor authentication (MFA), you will need to create an [app password](https://myaccount.google.com/apppasswords) and use this instead of your Gmail password. > :point_right: If you have multi-factor authentication (MFA), you will need to create an [app password](https://myaccount.google.com/apppasswords) and use this instead of your Gmail password.
src/config.ts
+2 -1
View File
@@ -16,7 +16,8 @@ const notifications = {
const page = { const page = {
height: 1920, height: 1920,
userAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36', userAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36',
width: 1080 width: 1080,
navigationTimeout: Number(process.env.PAGE_TIMEOUT) ?? 30000
}; };
const stores = process.env.STORES ?? 'nvidia'; const stores = process.env.STORES ?? 'nvidia';
src/index.ts
+1
View File
@@ -40,6 +40,7 @@ async function lookup(store: Store) {
for (const link of store.links) { for (const link of store.links) {
const browser = await puppeteer.launch(); const browser = await puppeteer.launch();
const page = await browser.newPage(); const page = await browser.newPage();
page.setDefaultNavigationTimeout(Config.page.navigationTimeout);
await page.setUserAgent(Config.page.userAgent); await page.setUserAgent(Config.page.userAgent);
await page.setViewport({ await page.setViewport({
height: Config.page.height, height: Config.page.height,