opelly/streetmerchant
1
0
Fork 0
mirror of https://github.com/opelly27/streetmerchant.git synced 2026-05-20 02:57:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(lookup): use random user agents (#1335)

Browse Source
This commit is contained in:
Jef LeCompte
2020-12-11 00:33:32 -05:00
committed by GitHub
parent 2115c0c906
commit b599c23b51
9 changed files with 116 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env-example
-1
View File
@@ -111,5 +111,4 @@ TWITTER_ACCESS_TOKEN_SECRET=""
TWITTER_CONSUMER_KEY="" TWITTER_CONSUMER_KEY=""
TWITTER_CONSUMER_SECRET="" TWITTER_CONSUMER_SECRET=""
TWITTER_TWEET_TAGS="" TWITTER_TWEET_TAGS=""
USER_AGENT=""
WEB_PORT="" WEB_PORT=""
docs/help/troubleshoot.md
+2 -10
View File
@@ -4,19 +4,11 @@
### Option 1 ### Option 1
If you're running into _a lot_ of captcha problems, be sure to update your user agent by searching ["what's my user agent" on Google](https://www.google.com/search?q=whats+my+user+agent). If you're running into problems, try running in headful mode: `HEADLESS="false"`.
![image](https://user-images.githubusercontent.com/12074633/101272427-07a88100-375a-11eb-9cb3-4e8783db6ae5.png)
You can update your user agent by using `USER_AGENT="your-result"`.
### Option 2
If you're _still_ running into problems, try running in headful mode: `HEADLESS="false"`.
This will open a browser and run streetmerchant. Note that this isn't a great solution for those running in a headless environment, i.e.: VPS, cloud, docker. Instead, it would be a good solution for those running on separate computer that won't be blocked by running in the background. This will open a browser and run streetmerchant. Note that this isn't a great solution for those running in a headless environment, i.e.: VPS, cloud, docker. Instead, it would be a good solution for those running on separate computer that won't be blocked by running in the background.
### Option 3 ### Option 2
As a last case scenario, use `PUPPETEER_EXECUTABLE_PATH`. This will use your computer's Chrome browser. You can run this is headless or headful mode. As a last case scenario, use `PUPPETEER_EXECUTABLE_PATH`. This will use your computer's Chrome browser. You can run this is headless or headful mode.
docs/reference/application.md
-1
View File
@@ -19,7 +19,6 @@
| `PROXY_ADDRESS` | IP Address or fqdn of proxy server | | `PROXY_ADDRESS` | IP Address or fqdn of proxy server |
| `PROXY_PORT` | TCP Port number on which the proxy is listening for connections. Default: `80` | | `PROXY_PORT` | TCP Port number on which the proxy is listening for connections. Default: `80` |
| `SCREENSHOT` | Capture screenshot of page if a card is found. Default: `true` | | `SCREENSHOT` | Capture screenshot of page if a card is found. Default: `true` |
| `USER_AGENT` | Custom User-Agents headers for HTTP requests. Newline separated, e.g.: `USER_AGENT_STRING1 \n USER_AGENT_STRING2`. Default: `Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36` |
| `WEB_PORT` | Starts a webserver to be able to control the bot while it is running. Setting this value starts this service. | | `WEB_PORT` | Starts a webserver to be able to control the bot while it is running. Setting this value starts this service. |
???+ info ???+ info
package-lock.json
Generated
+99 -49
View File
@@ -821,6 +821,11 @@
"@types/node": "*" "@types/node": "*"
} }
}, },
"@types/random-useragent": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/@types/random-useragent/-/random-useragent-0.3.0.tgz",
"integrity": "sha512-K6yHeQCCJJBvHeW1akRxXC8GnADsXmoRA2UdVEYITzD88U8Vk3lX+0c0A27j3CrAiic2I961glcnwScc9vUtWg=="
},
"@types/request": { "@types/request": {
"version": "2.48.5", "version": "2.48.5",
"resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.5.tgz", "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.5.tgz",
@@ -4906,9 +4911,9 @@
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
}, },
"ini": { "ini": {
"version": "1.3.5", "version": "1.3.7",
"resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.7.tgz",
"integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==", "integrity": "sha512-iKpRpXP+CrP2jyrxvg1kMUpXDyRUFDWurxbnVT1vQPx+Wz9uCYsMIqYuSBLV+PAaZG/d7kRLKRFc9oDMsH+mFQ==",
"dev": true "dev": true
}, },
"interpret": { "interpret": {
@@ -6521,14 +6526,14 @@
"integrity": "sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ==" "integrity": "sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ=="
}, },
"npm": { "npm": {
"version": "7.0.12", "version": "7.1.1",
"resolved": "https://registry.npmjs.org/npm/-/npm-7.0.12.tgz", "resolved": "https://registry.npmjs.org/npm/-/npm-7.1.1.tgz",
"integrity": "sha512-0MDC+wxNRTxp9yNZ1t8zEUjVvKTN8K2xGAQ9Cuert9NwrTk0GDw+8hl4OOVcMVDFSFaHopW3F+w+KnxLGZMwYw==", "integrity": "sha512-66kBbZSTTLGrAyNPou2dUoVjAxDBZZiJkMdukGac3tUJf33tq3c86zW0rVKN+j5pUdWyJKyK3DnzEmJIcq06mw==",
"requires": { "requires": {
"@npmcli/arborist": "^1.0.11", "@npmcli/arborist": "^2.0.0",
"@npmcli/ci-detect": "^1.2.0", "@npmcli/ci-detect": "^1.2.0",
"@npmcli/config": "^1.2.1", "@npmcli/config": "^1.2.4",
"@npmcli/run-script": "^1.8.0", "@npmcli/run-script": "^1.8.1",
"abbrev": "~1.1.1", "abbrev": "~1.1.1",
"ansicolors": "~0.3.2", "ansicolors": "~0.3.2",
"ansistyles": "~0.1.3", "ansistyles": "~0.1.3",
@@ -6547,20 +6552,20 @@
"graceful-fs": "^4.2.3", "graceful-fs": "^4.2.3",
"hosted-git-info": "^3.0.6", "hosted-git-info": "^3.0.6",
"inherits": "^2.0.4", "inherits": "^2.0.4",
"ini": "^1.3.5", "ini": "^1.3.6",
"init-package-json": "^2.0.1", "init-package-json": "^2.0.1",
"is-cidr": "^4.0.2", "is-cidr": "^4.0.2",
"leven": "^3.1.0", "leven": "^3.1.0",
"libnpmaccess": "^4.0.1", "libnpmaccess": "^4.0.1",
"libnpmfund": "^1.0.1", "libnpmfund": "^1.0.2",
"libnpmhook": "^6.0.1", "libnpmhook": "^6.0.1",
"libnpmorg": "^2.0.1", "libnpmorg": "^2.0.1",
"libnpmpack": "^2.0.0", "libnpmpack": "^2.0.0",
"libnpmpublish": "^4.0.0", "libnpmpublish": "^4.0.0",
"libnpmsearch": "^3.0.1", "libnpmsearch": "^3.1.0",
"libnpmteam": "^2.0.2", "libnpmteam": "^2.0.2",
"libnpmversion": "^1.0.7", "libnpmversion": "^1.0.7",
"make-fetch-happen": "^8.0.9", "make-fetch-happen": "^8.0.12",
"mkdirp": "^1.0.4", "mkdirp": "^1.0.4",
"mkdirp-infer-owner": "^2.0.0", "mkdirp-infer-owner": "^2.0.0",
"ms": "^2.1.2", "ms": "^2.1.2",
@@ -6582,7 +6587,7 @@
"read-package-json": "^3.0.0", "read-package-json": "^3.0.0",
"read-package-json-fast": "^1.2.1", "read-package-json-fast": "^1.2.1",
"rimraf": "^3.0.2", "rimraf": "^3.0.2",
"semver": "^7.3.2", "semver": "^7.3.4",
"sorted-object": "~2.0.1", "sorted-object": "~2.0.1",
"ssri": "^8.0.0", "ssri": "^8.0.0",
"tar": "^6.0.5", "tar": "^6.0.5",
@@ -6595,7 +6600,7 @@
}, },
"dependencies": { "dependencies": {
"@npmcli/arborist": { "@npmcli/arborist": {
"version": "1.0.11", "version": "2.0.0",
"bundled": true, "bundled": true,
"requires": { "requires": {
"@npmcli/installed-package-contents": "^1.0.5", "@npmcli/installed-package-contents": "^1.0.5",
@@ -6603,8 +6608,8 @@
"@npmcli/metavuln-calculator": "^1.0.0", "@npmcli/metavuln-calculator": "^1.0.0",
"@npmcli/move-file": "^1.0.1", "@npmcli/move-file": "^1.0.1",
"@npmcli/name-from-folder": "^1.0.1", "@npmcli/name-from-folder": "^1.0.1",
"@npmcli/node-gyp": "^1.0.0", "@npmcli/node-gyp": "^1.0.1",
"@npmcli/run-script": "^1.7.2", "@npmcli/run-script": "^1.8.1",
"bin-links": "^2.2.1", "bin-links": "^2.2.1",
"cacache": "^15.0.3", "cacache": "^15.0.3",
"common-ancestor-path": "^1.0.1", "common-ancestor-path": "^1.0.1",
@@ -6612,15 +6617,15 @@
"json-stringify-nice": "^1.1.1", "json-stringify-nice": "^1.1.1",
"mkdirp-infer-owner": "^2.0.0", "mkdirp-infer-owner": "^2.0.0",
"npm-install-checks": "^4.0.0", "npm-install-checks": "^4.0.0",
"npm-package-arg": "^8.0.0", "npm-package-arg": "^8.1.0",
"npm-pick-manifest": "^6.1.0", "npm-pick-manifest": "^6.1.0",
"pacote": "^11.1.10", "pacote": "^11.1.13",
"parse-conflict-json": "^1.1.1", "parse-conflict-json": "^1.1.1",
"promise-all-reject-late": "^1.0.0", "promise-all-reject-late": "^1.0.0",
"promise-call-limit": "^1.0.1", "promise-call-limit": "^1.0.1",
"read-package-json-fast": "^1.2.1", "read-package-json-fast": "^1.2.1",
"readdir-scoped-modules": "^1.1.0", "readdir-scoped-modules": "^1.1.0",
"semver": "^7.1.2", "semver": "^7.3.4",
"treeverse": "^1.0.4", "treeverse": "^1.0.4",
"walk-up-path": "^1.0.0" "walk-up-path": "^1.0.0"
} }
@@ -6630,12 +6635,13 @@
"bundled": true "bundled": true
}, },
"@npmcli/config": { "@npmcli/config": {
"version": "1.2.1", "version": "1.2.4",
"bundled": true, "bundled": true,
"requires": { "requires": {
"ini": "^1.3.5", "ini": "^1.3.5",
"mkdirp-infer-owner": "^2.0.0", "mkdirp-infer-owner": "^2.0.0",
"nopt": "^5.0.0", "nopt": "^5.0.0",
"semver": "^7.3.4",
"walk-up-path": "^1.0.0" "walk-up-path": "^1.0.0"
} }
}, },
@@ -6706,7 +6712,7 @@
} }
}, },
"@npmcli/run-script": { "@npmcli/run-script": {
"version": "1.8.0", "version": "1.8.1",
"bundled": true, "bundled": true,
"requires": { "requires": {
"@npmcli/node-gyp": "^1.0.0", "@npmcli/node-gyp": "^1.0.0",
@@ -6818,7 +6824,7 @@
"bundled": true "bundled": true
}, },
"aws4": { "aws4": {
"version": "1.10.1", "version": "1.11.0",
"bundled": true "bundled": true
}, },
"balanced-match": { "balanced-match": {
@@ -7024,10 +7030,16 @@
} }
}, },
"debug": { "debug": {
"version": "4.2.0", "version": "4.3.1",
"bundled": true, "bundled": true,
"requires": { "requires": {
"ms": "2.1.2" "ms": "2.1.2"
},
"dependencies": {
"ms": {
"version": "2.1.2",
"bundled": true
}
} }
}, },
"debuglog": { "debuglog": {
@@ -7133,6 +7145,10 @@
"version": "1.0.0", "version": "1.0.0",
"bundled": true "bundled": true
}, },
"function-bind": {
"version": "1.1.1",
"bundled": true
},
"gauge": { "gauge": {
"version": "2.7.4", "version": "2.7.4",
"bundled": true, "bundled": true,
@@ -7204,6 +7220,13 @@
"har-schema": "^2.0.0" "har-schema": "^2.0.0"
} }
}, },
"has": {
"version": "1.0.3",
"bundled": true,
"requires": {
"function-bind": "^1.1.1"
}
},
"has-flag": { "has-flag": {
"version": "4.0.0", "version": "4.0.0",
"bundled": true "bundled": true
@@ -7296,7 +7319,7 @@
"bundled": true "bundled": true
}, },
"ini": { "ini": {
"version": "1.3.5", "version": "1.3.6",
"bundled": true "bundled": true
}, },
"init-package-json": { "init-package-json": {
@@ -7328,6 +7351,13 @@
"cidr-regex": "^3.1.1" "cidr-regex": "^3.1.1"
} }
}, },
"is-core-module": {
"version": "2.2.0",
"bundled": true,
"requires": {
"has": "^1.0.3"
}
},
"is-fullwidth-code-point": { "is-fullwidth-code-point": {
"version": "2.0.0", "version": "2.0.0",
"bundled": true "bundled": true
@@ -7413,10 +7443,10 @@
} }
}, },
"libnpmfund": { "libnpmfund": {
"version": "1.0.1", "version": "1.0.2",
"bundled": true, "bundled": true,
"requires": { "requires": {
"@npmcli/arborist": "^0.0.33 || ^1.x" "@npmcli/arborist": "^2.0.0"
} }
}, },
"libnpmhook": { "libnpmhook": {
@@ -7456,7 +7486,7 @@
} }
}, },
"libnpmsearch": { "libnpmsearch": {
"version": "3.0.1", "version": "3.1.0",
"bundled": true, "bundled": true,
"requires": { "requires": {
"npm-registry-fetch": "^9.0.0" "npm-registry-fetch": "^9.0.0"
@@ -7489,21 +7519,21 @@
} }
}, },
"make-fetch-happen": { "make-fetch-happen": {
"version": "8.0.10", "version": "8.0.12",
"bundled": true, "bundled": true,
"requires": { "requires": {
"agentkeepalive": "^4.1.0", "agentkeepalive": "^4.1.3",
"cacache": "^15.0.0", "cacache": "^15.0.5",
"http-cache-semantics": "^4.0.4", "http-cache-semantics": "^4.1.0",
"http-proxy-agent": "^4.0.1", "http-proxy-agent": "^4.0.1",
"https-proxy-agent": "^5.0.0", "https-proxy-agent": "^5.0.0",
"is-lambda": "^1.0.1", "is-lambda": "^1.0.1",
"lru-cache": "^6.0.0", "lru-cache": "^6.0.0",
"minipass": "^3.1.3", "minipass": "^3.1.3",
"minipass-collect": "^1.0.2", "minipass-collect": "^1.0.2",
"minipass-fetch": "^1.3.0", "minipass-fetch": "^1.3.2",
"minipass-flush": "^1.0.5", "minipass-flush": "^1.0.5",
"minipass-pipeline": "^1.2.2", "minipass-pipeline": "^1.2.4",
"promise-retry": "^1.1.1", "promise-retry": "^1.1.1",
"socks-proxy-agent": "^5.0.0", "socks-proxy-agent": "^5.0.0",
"ssri": "^8.0.0" "ssri": "^8.0.0"
@@ -7602,7 +7632,7 @@
} }
}, },
"ms": { "ms": {
"version": "2.1.2", "version": "2.1.3",
"bundled": true "bundled": true
}, },
"mute-stream": { "mute-stream": {
@@ -7935,6 +7965,14 @@
"uuid": "^3.3.2" "uuid": "^3.3.2"
}, },
"dependencies": { "dependencies": {
"tough-cookie": {
"version": "2.5.0",
"bundled": true,
"requires": {
"psl": "^1.1.28",
"punycode": "^2.1.1"
}
},
"uuid": { "uuid": {
"version": "3.4.0", "version": "3.4.0",
"bundled": true "bundled": true
@@ -7942,9 +7980,10 @@
} }
}, },
"resolve": { "resolve": {
"version": "1.17.0", "version": "1.19.0",
"bundled": true, "bundled": true,
"requires": { "requires": {
"is-core-module": "^2.1.0",
"path-parse": "^1.0.6" "path-parse": "^1.0.6"
} }
}, },
@@ -7968,8 +8007,11 @@
"bundled": true "bundled": true
}, },
"semver": { "semver": {
"version": "7.3.2", "version": "7.3.4",
"bundled": true "bundled": true,
"requires": {
"lru-cache": "^6.0.0"
}
}, },
"set-blocking": { "set-blocking": {
"version": "2.0.0", "version": "2.0.0",
@@ -7984,7 +8026,7 @@
"bundled": true "bundled": true
}, },
"socks": { "socks": {
"version": "2.4.4", "version": "2.5.1",
"bundled": true, "bundled": true,
"requires": { "requires": {
"ip": "^1.1.5", "ip": "^1.1.5",
@@ -8025,7 +8067,7 @@
} }
}, },
"spdx-license-ids": { "spdx-license-ids": {
"version": "3.0.6", "version": "3.0.7",
"bundled": true "bundled": true
}, },
"sshpk": { "sshpk": {
@@ -8116,14 +8158,6 @@
"version": "1.3.0", "version": "1.3.0",
"bundled": true "bundled": true
}, },
"tough-cookie": {
"version": "2.5.0",
"bundled": true,
"requires": {
"psl": "^1.1.28",
"punycode": "^2.1.1"
}
},
"treeverse": { "treeverse": {
"version": "1.0.4", "version": "1.0.4",
"bundled": true "bundled": true
@@ -8172,7 +8206,7 @@
"bundled": true "bundled": true
}, },
"uuid": { "uuid": {
"version": "8.3.1", "version": "8.3.2",
"bundled": true "bundled": true
}, },
"validate-npm-package-license": { "validate-npm-package-license": {
@@ -9251,6 +9285,22 @@
"integrity": "sha512-ARhCpm70fzdcvNQfPoy49IaanKkTlRWF2JMzqhcJbhSFRZv7nPTvZJdcY7301IPmvW+/p0RgIWnQDLJxifsQ7g==", "integrity": "sha512-ARhCpm70fzdcvNQfPoy49IaanKkTlRWF2JMzqhcJbhSFRZv7nPTvZJdcY7301IPmvW+/p0RgIWnQDLJxifsQ7g==",
"dev": true "dev": true
}, },
"random-seed": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/random-seed/-/random-seed-0.3.0.tgz",
"integrity": "sha1-2UXy4fOPSejViRNDG4v2u5N1Vs0=",
"requires": {
"json-stringify-safe": "^5.0.1"
}
},
"random-useragent": {
"version": "0.5.0",
"resolved": "https://registry.npmjs.org/random-useragent/-/random-useragent-0.5.0.tgz",
"integrity": "sha512-FUMkqVdZeoSff5tErNL3FFGYXElDWZ1bEuedhm5u9MdCFwANriJWbHvDRYrLTOzp/fBsBGu5J1cWtDgifa97aQ==",
"requires": {
"random-seed": "^0.3.0"
}
},
"randombytes": { "randombytes": {
"version": "2.1.0", "version": "2.1.0",
"resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz",
package.json
+2
View File
@@ -27,6 +27,7 @@
"dependencies": { "dependencies": {
"@jef/pushbullet": "^2.4.3", "@jef/pushbullet": "^2.4.3",
"@slack/web-api": "^5.14.0", "@slack/web-api": "^5.14.0",
"@types/random-useragent": "^0.3.0",
"chalk": "^4.1.0", "chalk": "^4.1.0",
"cheerio": "^1.0.0-rc.3", "cheerio": "^1.0.0-rc.3",
"discord.js": "^12.5.1", "discord.js": "^12.5.1",
@@ -47,6 +48,7 @@
"puppeteer-extra-plugin-stealth": "^2.6.5", "puppeteer-extra-plugin-stealth": "^2.6.5",
"puppeteer-page-proxy": "^1.2.8", "puppeteer-page-proxy": "^1.2.8",
"pushover-notifications": "^1.2.2", "pushover-notifications": "^1.2.2",
"random-useragent": "^0.5.0",
"twilio": "^3.53.0", "twilio": "^3.53.0",
"twitch": "^4.3.5", "twitch": "^4.3.5",
"twitch-auth": "^4.3.5", "twitch-auth": "^4.3.5",
src/config.ts
-3
View File
@@ -307,9 +307,6 @@ const page = {
inStockWaitTime: envOrNumber(process.env.IN_STOCK_WAIT_TIME), inStockWaitTime: envOrNumber(process.env.IN_STOCK_WAIT_TIME),
screenshot: envOrBoolean(process.env.SCREENSHOT), screenshot: envOrBoolean(process.env.SCREENSHOT),
timeout: envOrNumber(process.env.PAGE_TIMEOUT, 30000), timeout: envOrNumber(process.env.PAGE_TIMEOUT, 30000),
userAgents: envOrArray(process.env.USER_AGENT, [
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'
]),
width: 1920 width: 1920
}; };
src/store/lookup.ts
+1 -1
View File
@@ -162,7 +162,7 @@ async function lookup(browser: Browser, store: Store) {
const page = await context.newPage(); const page = await context.newPage();
page.setDefaultNavigationTimeout(config.page.timeout); page.setDefaultNavigationTimeout(config.page.timeout);
await page.setUserAgent(getRandomUserAgent()); await page.setUserAgent(await getRandomUserAgent(browser));
let adBlockRequestHandler: any; let adBlockRequestHandler: any;
let pageProxy; let pageProxy;
src/store/model/amazon.ts
+1 -1
View File
@@ -50,7 +50,7 @@ export const Amazon: Store = {
brand: 'gigabyte', brand: 'gigabyte',
cartUrl: cartUrl:
'https://www.amazon.com/gp/aws/cart/add.html?ASIN.1=B08NYPLXPJ&Quantity.1=1', 'https://www.amazon.com/gp/aws/cart/add.html?ASIN.1=B08NYPLXPJ&Quantity.1=1',
model: 'gaming oc', model: 'gaming oc pro',
series: '3060ti', series: '3060ti',
url: 'https://www.amazon.com/dp/B08NYPLXPJ' url: 'https://www.amazon.com/dp/B08NYPLXPJ'
}, },
src/util.ts
+11 -5
View File
@@ -2,6 +2,7 @@ import {Browser, Page, Response} from 'puppeteer';
import {StatusCodeRangeArray, Store} from './store/model'; import {StatusCodeRangeArray, Store} from './store/model';
import {config} from './config'; import {config} from './config';
import {disableBlockerInPage} from './adblocker'; import {disableBlockerInPage} from './adblocker';
import {getRandom} from 'random-useragent';
import {logger} from './logger'; import {logger} from './logger';
export function getSleepTime(store: Store) { export function getSleepTime(store: Store) {
@@ -57,7 +58,7 @@ export async function usingPage<T>(
): Promise<T> { ): Promise<T> {
const page = await browser.newPage(); const page = await browser.newPage();
page.setDefaultNavigationTimeout(config.page.timeout); page.setDefaultNavigationTimeout(config.page.timeout);
await page.setUserAgent(getRandomUserAgent()); await page.setUserAgent(await getRandomUserAgent(browser));
try { try {
return await cb(page, browser); return await cb(page, browser);
@@ -78,8 +79,13 @@ export async function closePage(page: Page) {
await page.close(); await page.close();
} }
export function getRandomUserAgent(): string { export async function getRandomUserAgent(browser: Browser): Promise<string> {
return config.page.userAgents[ const userAgent =
Math.floor(Math.random() * config.page.userAgents.length) getRandom((ua) => {
]; return ua.browserName === 'Chrome' && ua.browserVersion > '20';
}) ?? (await browser.userAgent());
logger.debug('user agent', userAgent);
return userAgent;
} }