opelly/streetmerchant
1
0
Fork 0
mirror of https://github.com/opelly27/streetmerchant.git synced 2026-05-20 06:27:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add support for multiple browser user agents (#547)

Browse Source 
Co-authored-by: Pawel Grzesik <p.grzesik@francotyp.com>
This commit is contained in:
SesioN
2020-10-20 20:03:38 +02:00
committed by GitHub
parent 68ff5bf836
commit 10a81dcbd5
4 changed files with 12 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+1 -1
View File
@@ -141,7 +141,7 @@ Here is a list of variables that you can use to customize your newly copied `.en
| `TWITTER_CONSUMER_KEY` | Twitter Consumer Key | Generate all Twitter keys at: https://developer.twitter.com/ | | `TWITTER_CONSUMER_KEY` | Twitter Consumer Key | Generate all Twitter keys at: https://developer.twitter.com/ |
| `TWITTER_CONSUMER_SECRET` | Twitter Consumer Secret | | | `TWITTER_CONSUMER_SECRET` | Twitter Consumer Secret | |
| `TWITTER_TWEET_TAGS` | Optional list of hashtags to append to the tweet message | E.g.: `#nvidia #nvidiastock` | | `TWITTER_TWEET_TAGS` | Optional list of hashtags to append to the tweet message | E.g.: `#nvidia #nvidiastock` |
| `USER_AGENT` | Custom User-Agent header for HTTP requests | Default: `Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36` | | `USER_AGENT` | Custom User-Agents headers for HTTP requests | Newline separated, E.g.: `USER_AGENT_STRING1 \n USER_AGENT_STRING2` | | Default: `Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36` |
> :point_right: If you have multi-factor authentication (MFA), you will need to create an [app password](https://myaccount.google.com/apppasswords) and use this instead of your Gmail password. > :point_right: If you have multi-factor authentication (MFA), you will need to create an [app password](https://myaccount.google.com/apppasswords) and use this instead of your Gmail password.
src/config.ts
+4 -2
View File
@@ -14,7 +14,9 @@ config_({path: path.resolve(__dirname, '../.env')});
* @param array Default array. If not set, is `[]`. * @param array Default array. If not set, is `[]`.
*/ */
function envOrArray(environment: string | undefined, array?: string[]): string[] { function envOrArray(environment: string | undefined, array?: string[]): string[] {
return (environment ? environment.split(',') : (array ?? [])).map(s => s.trim()); return (environment ? (
environment.includes('\n') ? environment.split('\n') : environment.split(',')
) : (array ?? [])).map(s => s.trim());
} }
/** /**
@@ -205,7 +207,7 @@ const page = {
inStockWaitTime: envOrNumber(process.env.IN_STOCK_WAIT_TIME), inStockWaitTime: envOrNumber(process.env.IN_STOCK_WAIT_TIME),
screenshot: envOrBoolean(process.env.SCREENSHOT), screenshot: envOrBoolean(process.env.SCREENSHOT),
timeout: envOrNumber(process.env.PAGE_TIMEOUT, 30000), timeout: envOrNumber(process.env.PAGE_TIMEOUT, 30000),
userAgent: envOrString(process.env.USER_AGENT, 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'), userAgents: envOrArray(process.env.USER_AGENT, ['Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36']),
width: 1920 width: 1920
}; };
src/store/lookup.ts
+2 -2
View File
@@ -2,7 +2,7 @@ import {Browser, Page, Response} from 'puppeteer';
import {Link, Store} from './model'; import {Link, Store} from './model';
import {Print, logger} from '../logger'; import {Print, logger} from '../logger';
import {Selector, cardPrice, pageIncludesLabels} from './includes-labels'; import {Selector, cardPrice, pageIncludesLabels} from './includes-labels';
import {closePage, delay, getSleepTime, isStatusCodeInRange} from '../util'; import {closePage, delay, getRandomUserAgent, getSleepTime, isStatusCodeInRange} from '../util';
import {config} from '../config'; import {config} from '../config';
import {disableBlockerInPage} from '../adblocker'; import {disableBlockerInPage} from '../adblocker';
import {fetchLinks} from './fetch-links'; import {fetchLinks} from './fetch-links';
@@ -38,7 +38,7 @@ async function lookup(browser: Browser, store: Store) {
const context = (config.browser.isIncognito ? await browser.createIncognitoBrowserContext() : browser.defaultBrowserContext()); const context = (config.browser.isIncognito ? await browser.createIncognitoBrowserContext() : browser.defaultBrowserContext());
const page = (config.browser.isIncognito ? await context.newPage() : await browser.newPage()); const page = (config.browser.isIncognito ? await context.newPage() : await browser.newPage());
page.setDefaultNavigationTimeout(config.page.timeout); page.setDefaultNavigationTimeout(config.page.timeout);
await page.setUserAgent(config.page.userAgent); await page.setUserAgent(getRandomUserAgent());
if (store.disableAdBlocker) { if (store.disableAdBlocker) {
try { try {
src/util.ts
+5 -1
View File
@@ -48,7 +48,7 @@ export async function usingResponse<T>(
export async function usingPage<T>(browser: Browser, cb: (page: Page, browser: Browser) => Promise<T>): Promise<T> { export async function usingPage<T>(browser: Browser, cb: (page: Page, browser: Browser) => Promise<T>): Promise<T> {
const page = await browser.newPage(); const page = await browser.newPage();
page.setDefaultNavigationTimeout(config.page.timeout); page.setDefaultNavigationTimeout(config.page.timeout);
await page.setUserAgent(config.page.userAgent); await page.setUserAgent(getRandomUserAgent());
try { try {
return await cb(page, browser); return await cb(page, browser);
@@ -68,3 +68,7 @@ export async function closePage(page: Page) {
await page.close(); await page.close();
} }
export function getRandomUserAgent(): string {
return config.page.userAgents[Math.floor(Math.random() * config.page.userAgents.length)];
}