opelly/streetmerchant
1
0
Fork 0
mirror of https://github.com/opelly27/streetmerchant.git synced 2026-05-20 02:57:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(lookup): use random user agents (#1335)

Browse Source
This commit is contained in:
Jef LeCompte
2020-12-11 00:33:32 -05:00
committed by GitHub
parent 2115c0c906
commit b599c23b51
9 changed files with 116 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env-example
-1
View File
@@ -111,5 +111,4 @@ TWITTER_ACCESS_TOKEN_SECRET=""
TWITTER_CONSUMER_KEY=""
TWITTER_CONSUMER_SECRET=""
TWITTER_TWEET_TAGS=""
USER_AGENT=""
WEB_PORT=""
docs/help/troubleshoot.md
+2 -10
View File
@@ -4,19 +4,11 @@
### Option 1
If you're running into _a lot_ of captcha problems, be sure to update your user agent by searching ["what's my user agent" on Google](https://www.google.com/search?q=whats+my+user+agent).
![image](https://user-images.githubusercontent.com/12074633/101272427-07a88100-375a-11eb-9cb3-4e8783db6ae5.png)
You can update your user agent by using `USER_AGENT="your-result"`.
### Option 2
If you're _still_ running into problems, try running in headful mode: `HEADLESS="false"`.
If you're running into problems, try running in headful mode: `HEADLESS="false"`.
This will open a browser and run streetmerchant. Note that this isn't a great solution for those running in a headless environment, i.e.: VPS, cloud, docker. Instead, it would be a good solution for those running on separate computer that won't be blocked by running in the background.
### Option 3
### Option 2
As a last case scenario, use `PUPPETEER_EXECUTABLE_PATH`. This will use your computer's Chrome browser. You can run this is headless or headful mode.
docs/reference/application.md
-1
View File
@@ -19,7 +19,6 @@
| `PROXY_ADDRESS` | IP Address or fqdn of proxy server |
| `PROXY_PORT` | TCP Port number on which the proxy is listening for connections. Default: `80` |
| `SCREENSHOT` | Capture screenshot of page if a card is found. Default: `true` |
| `USER_AGENT` | Custom User-Agents headers for HTTP requests. Newline separated, e.g.: `USER_AGENT_STRING1 \n USER_AGENT_STRING2`. Default: `Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36` |
| `WEB_PORT` | Starts a webserver to be able to control the bot while it is running. Setting this value starts this service. |
???+ info
package-lock.json
Generated
+99 -49
View File
@@ -821,6 +821,11 @@
"@types/node": "*"
}
},
"@types/random-useragent": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/@types/random-useragent/-/random-useragent-0.3.0.tgz",
"integrity": "sha512-K6yHeQCCJJBvHeW1akRxXC8GnADsXmoRA2UdVEYITzD88U8Vk3lX+0c0A27j3CrAiic2I961glcnwScc9vUtWg=="
},
"@types/request": {
"version": "2.48.5",
"resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.5.tgz",
@@ -4906,9 +4911,9 @@
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
},
"ini": {
"version": "1.3.5",
"resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz",
"integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==",
"version": "1.3.7",
"resolved": "https://registry.npmjs.org/ini/-/ini-1.3.7.tgz",
"integrity": "sha512-iKpRpXP+CrP2jyrxvg1kMUpXDyRUFDWurxbnVT1vQPx+Wz9uCYsMIqYuSBLV+PAaZG/d7kRLKRFc9oDMsH+mFQ==",
"dev": true
},
"interpret": {
@@ -6521,14 +6526,14 @@
"integrity": "sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ=="
},
"npm": {
"version": "7.0.12",
"resolved": "https://registry.npmjs.org/npm/-/npm-7.0.12.tgz",
"integrity": "sha512-0MDC+wxNRTxp9yNZ1t8zEUjVvKTN8K2xGAQ9Cuert9NwrTk0GDw+8hl4OOVcMVDFSFaHopW3F+w+KnxLGZMwYw==",
"version": "7.1.1",
"resolved": "https://registry.npmjs.org/npm/-/npm-7.1.1.tgz",
"integrity": "sha512-66kBbZSTTLGrAyNPou2dUoVjAxDBZZiJkMdukGac3tUJf33tq3c86zW0rVKN+j5pUdWyJKyK3DnzEmJIcq06mw==",
"requires": {
"@npmcli/arborist": "^1.0.11",
"@npmcli/arborist": "^2.0.0",
"@npmcli/ci-detect": "^1.2.0",
"@npmcli/config": "^1.2.1",
"@npmcli/run-script": "^1.8.0",
"@npmcli/config": "^1.2.4",
"@npmcli/run-script": "^1.8.1",
"abbrev": "~1.1.1",
"ansicolors": "~0.3.2",
"ansistyles": "~0.1.3",
@@ -6547,20 +6552,20 @@
"graceful-fs": "^4.2.3",
"hosted-git-info": "^3.0.6",
"inherits": "^2.0.4",
"ini": "^1.3.5",
"ini": "^1.3.6",
"init-package-json": "^2.0.1",
"is-cidr": "^4.0.2",
"leven": "^3.1.0",
"libnpmaccess": "^4.0.1",
"libnpmfund": "^1.0.1",
"libnpmfund": "^1.0.2",
"libnpmhook": "^6.0.1",
"libnpmorg": "^2.0.1",
"libnpmpack": "^2.0.0",
"libnpmpublish": "^4.0.0",
"libnpmsearch": "^3.0.1",
"libnpmsearch": "^3.1.0",
"libnpmteam": "^2.0.2",
"libnpmversion": "^1.0.7",
"make-fetch-happen": "^8.0.9",
"make-fetch-happen": "^8.0.12",
"mkdirp": "^1.0.4",
"mkdirp-infer-owner": "^2.0.0",
"ms": "^2.1.2",
@@ -6582,7 +6587,7 @@
"read-package-json": "^3.0.0",
"read-package-json-fast": "^1.2.1",
"rimraf": "^3.0.2",
"semver": "^7.3.2",
"semver": "^7.3.4",
"sorted-object": "~2.0.1",
"ssri": "^8.0.0",
"tar": "^6.0.5",
@@ -6595,7 +6600,7 @@
},
"dependencies": {
"@npmcli/arborist": {
"version": "1.0.11",
"version": "2.0.0",
"bundled": true,
"requires": {
"@npmcli/installed-package-contents": "^1.0.5",
@@ -6603,8 +6608,8 @@
"@npmcli/metavuln-calculator": "^1.0.0",
"@npmcli/move-file": "^1.0.1",
"@npmcli/name-from-folder": "^1.0.1",
"@npmcli/node-gyp": "^1.0.0",
"@npmcli/run-script": "^1.7.2",
"@npmcli/node-gyp": "^1.0.1",
"@npmcli/run-script": "^1.8.1",
"bin-links": "^2.2.1",
"cacache": "^15.0.3",
"common-ancestor-path": "^1.0.1",
@@ -6612,15 +6617,15 @@
"json-stringify-nice": "^1.1.1",
"mkdirp-infer-owner": "^2.0.0",
"npm-install-checks": "^4.0.0",
"npm-package-arg": "^8.0.0",
"npm-package-arg": "^8.1.0",
"npm-pick-manifest": "^6.1.0",
"pacote": "^11.1.10",
"pacote": "^11.1.13",
"parse-conflict-json": "^1.1.1",
"promise-all-reject-late": "^1.0.0",
"promise-call-limit": "^1.0.1",
"read-package-json-fast": "^1.2.1",
"readdir-scoped-modules": "^1.1.0",
"semver": "^7.1.2",
"semver": "^7.3.4",
"treeverse": "^1.0.4",
"walk-up-path": "^1.0.0"
}
@@ -6630,12 +6635,13 @@
"bundled": true
},
"@npmcli/config": {
"version": "1.2.1",
"version": "1.2.4",
"bundled": true,
"requires": {
"ini": "^1.3.5",
"mkdirp-infer-owner": "^2.0.0",
"nopt": "^5.0.0",
"semver": "^7.3.4",
"walk-up-path": "^1.0.0"
}
},
@@ -6706,7 +6712,7 @@
}
},
"@npmcli/run-script": {
"version": "1.8.0",
"version": "1.8.1",
"bundled": true,
"requires": {
"@npmcli/node-gyp": "^1.0.0",
@@ -6818,7 +6824,7 @@
"bundled": true
},
"aws4": {
"version": "1.10.1",
"version": "1.11.0",
"bundled": true
},
"balanced-match": {
@@ -7024,10 +7030,16 @@
}
},
"debug": {
"version": "4.2.0",
"version": "4.3.1",
"bundled": true,
"requires": {
"ms": "2.1.2"
},
"dependencies": {
"ms": {
"version": "2.1.2",
"bundled": true
}
}
},
"debuglog": {
@@ -7133,6 +7145,10 @@
"version": "1.0.0",
"bundled": true
},
"function-bind": {
"version": "1.1.1",
"bundled": true
},
"gauge": {
"version": "2.7.4",
"bundled": true,
@@ -7204,6 +7220,13 @@
"har-schema": "^2.0.0"
}
},
"has": {
"version": "1.0.3",
"bundled": true,
"requires": {
"function-bind": "^1.1.1"
}
},
"has-flag": {
"version": "4.0.0",
"bundled": true
@@ -7296,7 +7319,7 @@
"bundled": true
},
"ini": {
"version": "1.3.5",
"version": "1.3.6",
"bundled": true
},
"init-package-json": {
@@ -7328,6 +7351,13 @@
"cidr-regex": "^3.1.1"
}
},
"is-core-module": {
"version": "2.2.0",
"bundled": true,
"requires": {
"has": "^1.0.3"
}
},
"is-fullwidth-code-point": {
"version": "2.0.0",
"bundled": true
@@ -7413,10 +7443,10 @@
}
},
"libnpmfund": {
"version": "1.0.1",
"version": "1.0.2",
"bundled": true,
"requires": {
"@npmcli/arborist": "^0.0.33 || ^1.x"
"@npmcli/arborist": "^2.0.0"
}
},
"libnpmhook": {
@@ -7456,7 +7486,7 @@
}
},
"libnpmsearch": {
"version": "3.0.1",
"version": "3.1.0",
"bundled": true,
"requires": {
"npm-registry-fetch": "^9.0.0"
@@ -7489,21 +7519,21 @@
}
},
"make-fetch-happen": {
"version": "8.0.10",
"version": "8.0.12",
"bundled": true,
"requires": {
"agentkeepalive": "^4.1.0",
"cacache": "^15.0.0",
"http-cache-semantics": "^4.0.4",
"agentkeepalive": "^4.1.3",
"cacache": "^15.0.5",
"http-cache-semantics": "^4.1.0",
"http-proxy-agent": "^4.0.1",
"https-proxy-agent": "^5.0.0",
"is-lambda": "^1.0.1",
"lru-cache": "^6.0.0",
"minipass": "^3.1.3",
"minipass-collect": "^1.0.2",
"minipass-fetch": "^1.3.0",
"minipass-fetch": "^1.3.2",
"minipass-flush": "^1.0.5",
"minipass-pipeline": "^1.2.2",
"minipass-pipeline": "^1.2.4",
"promise-retry": "^1.1.1",
"socks-proxy-agent": "^5.0.0",
"ssri": "^8.0.0"
@@ -7602,7 +7632,7 @@
}
},
"ms": {
"version": "2.1.2",
"version": "2.1.3",
"bundled": true
},
"mute-stream": {
@@ -7935,6 +7965,14 @@
"uuid": "^3.3.2"
},
"dependencies": {
"tough-cookie": {
"version": "2.5.0",
"bundled": true,
"requires": {
"psl": "^1.1.28",
"punycode": "^2.1.1"
}
},
"uuid": {
"version": "3.4.0",
"bundled": true
@@ -7942,9 +7980,10 @@
}
},
"resolve": {
"version": "1.17.0",
"version": "1.19.0",
"bundled": true,
"requires": {
"is-core-module": "^2.1.0",
"path-parse": "^1.0.6"
}
},
@@ -7968,8 +8007,11 @@
"bundled": true
},
"semver": {
"version": "7.3.2",
"bundled": true
"version": "7.3.4",
"bundled": true,
"requires": {
"lru-cache": "^6.0.0"
}
},
"set-blocking": {
"version": "2.0.0",
@@ -7984,7 +8026,7 @@
"bundled": true
},
"socks": {
"version": "2.4.4",
"version": "2.5.1",
"bundled": true,
"requires": {
"ip": "^1.1.5",
@@ -8025,7 +8067,7 @@
}
},
"spdx-license-ids": {
"version": "3.0.6",
"version": "3.0.7",
"bundled": true
},
"sshpk": {
@@ -8116,14 +8158,6 @@
"version": "1.3.0",
"bundled": true
},
"tough-cookie": {
"version": "2.5.0",
"bundled": true,
"requires": {
"psl": "^1.1.28",
"punycode": "^2.1.1"
}
},
"treeverse": {
"version": "1.0.4",
"bundled": true
@@ -8172,7 +8206,7 @@
"bundled": true
},
"uuid": {
"version": "8.3.1",
"version": "8.3.2",
"bundled": true
},
"validate-npm-package-license": {
@@ -9251,6 +9285,22 @@
"integrity": "sha512-ARhCpm70fzdcvNQfPoy49IaanKkTlRWF2JMzqhcJbhSFRZv7nPTvZJdcY7301IPmvW+/p0RgIWnQDLJxifsQ7g==",
"dev": true
},
"random-seed": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/random-seed/-/random-seed-0.3.0.tgz",
"integrity": "sha1-2UXy4fOPSejViRNDG4v2u5N1Vs0=",
"requires": {
"json-stringify-safe": "^5.0.1"
}
},
"random-useragent": {
"version": "0.5.0",
"resolved": "https://registry.npmjs.org/random-useragent/-/random-useragent-0.5.0.tgz",
"integrity": "sha512-FUMkqVdZeoSff5tErNL3FFGYXElDWZ1bEuedhm5u9MdCFwANriJWbHvDRYrLTOzp/fBsBGu5J1cWtDgifa97aQ==",
"requires": {
"random-seed": "^0.3.0"
}
},
"randombytes": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz",
package.json
+2
View File
@@ -27,6 +27,7 @@
"dependencies": {
"@jef/pushbullet": "^2.4.3",
"@slack/web-api": "^5.14.0",
"@types/random-useragent": "^0.3.0",
"chalk": "^4.1.0",
"cheerio": "^1.0.0-rc.3",
"discord.js": "^12.5.1",
@@ -47,6 +48,7 @@
"puppeteer-extra-plugin-stealth": "^2.6.5",
"puppeteer-page-proxy": "^1.2.8",
"pushover-notifications": "^1.2.2",
"random-useragent": "^0.5.0",
"twilio": "^3.53.0",
"twitch": "^4.3.5",
"twitch-auth": "^4.3.5",
src/config.ts
-3
View File
@@ -307,9 +307,6 @@ const page = {
inStockWaitTime: envOrNumber(process.env.IN_STOCK_WAIT_TIME),
screenshot: envOrBoolean(process.env.SCREENSHOT),
timeout: envOrNumber(process.env.PAGE_TIMEOUT, 30000),
userAgents: envOrArray(process.env.USER_AGENT, [
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'
]),
width: 1920
};
src/store/lookup.ts
+1 -1
View File
@@ -162,7 +162,7 @@ async function lookup(browser: Browser, store: Store) {
const page = await context.newPage();
page.setDefaultNavigationTimeout(config.page.timeout);
await page.setUserAgent(getRandomUserAgent());
await page.setUserAgent(await getRandomUserAgent(browser));
let adBlockRequestHandler: any;
let pageProxy;
src/store/model/amazon.ts
+1 -1
View File
@@ -50,7 +50,7 @@ export const Amazon: Store = {
brand: 'gigabyte',
cartUrl:
'https://www.amazon.com/gp/aws/cart/add.html?ASIN.1=B08NYPLXPJ&Quantity.1=1',
model: 'gaming oc',
model: 'gaming oc pro',
series: '3060ti',
url: 'https://www.amazon.com/dp/B08NYPLXPJ'
},
src/util.ts
+11 -5
View File
@@ -2,6 +2,7 @@ import {Browser, Page, Response} from 'puppeteer';
import {StatusCodeRangeArray, Store} from './store/model';
import {config} from './config';
import {disableBlockerInPage} from './adblocker';
import {getRandom} from 'random-useragent';
import {logger} from './logger';
export function getSleepTime(store: Store) {
@@ -57,7 +58,7 @@ export async function usingPage<T>(
): Promise<T> {
const page = await browser.newPage();
page.setDefaultNavigationTimeout(config.page.timeout);
await page.setUserAgent(getRandomUserAgent());
await page.setUserAgent(await getRandomUserAgent(browser));
try {
return await cb(page, browser);
@@ -78,8 +79,13 @@ export async function closePage(page: Page) {
await page.close();
}
export function getRandomUserAgent(): string {
return config.page.userAgents[
Math.floor(Math.random() * config.page.userAgents.length)
];
export async function getRandomUserAgent(browser: Browser): Promise<string> {
const userAgent =
getRandom((ua) => {
return ua.browserName === 'Chrome' && ua.browserVersion > '20';
}) ?? (await browser.userAgent());
logger.debug('user agent', userAgent);
return userAgent;
}