opelly/streetmerchant
1
0
Fork 0
mirror of https://github.com/opelly27/streetmerchant.git synced 2026-05-20 04:07:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add proxy rotation (settable per store) (#1026)

Browse Source
This commit is contained in:
Mark Dietzer
2020-12-01 19:25:48 -08:00
committed by GitHub
parent 65df944973
commit 490d44e1fd
8 changed files with 498 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package-lock.json
Generated
+304 -7
View File
@@ -590,12 +590,28 @@
"defer-to-connect": "^1.0.1"
}
},
"@tootallnate/once": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz",
"integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw=="
},
"@types/async": {
"version": "3.2.4",
"resolved": "https://registry.npmjs.org/@types/async/-/async-3.2.4.tgz",
"integrity": "sha512-2c14HTaCn6tlb7bnqJGeH5JJGHZmyrkDD3EYe1NYBYjuY4+2EmMg8pjau2HvVz8+ANmO+DieV95H8PlzbK383g==",
"dev": true
},
"@types/cacheable-request": {
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.1.tgz",
"integrity": "sha512-ykFq2zmBGOCbpIXtoVbz4SKY5QriWPh3AjyU4G74RYbtt5yOc5OfaY75ftjg7mikMOla1CTGpX3lLbuJh8DTrQ==",
"requires": {
"@types/http-cache-semantics": "*",
"@types/keyv": "*",
"@types/node": "*",
"@types/responselike": "*"
}
},
"@types/caseless": {
"version": "0.12.2",
"resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.2.tgz",
@@ -683,6 +699,11 @@
"resolved": "https://registry.npmjs.org/@types/har-format/-/har-format-1.2.5.tgz",
"integrity": "sha512-IG8AE1m2pWtPqQ7wXhFhy6Q59bwwnLwO36v5Rit2FrbXCIp8Sk8E2PfUCreyrdo17STwFSKDAkitVuVYbpEHvQ=="
},
"@types/http-cache-semantics": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.0.tgz",
"integrity": "sha512-c3Xy026kOF7QOTn00hbIllV1dLR9hG9NkSrLQgCVs8NF6sBU+VGWjD3wLPhmh1TYAc7ugCFsvHYMN4VcBN1U1A=="
},
"@types/is-stream": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@types/is-stream/-/is-stream-1.1.0.tgz",
@@ -703,6 +724,14 @@
"integrity": "sha1-7ihweulOEdK4J7y+UnC86n8+ce4=",
"dev": true
},
"@types/keyv": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/@types/keyv/-/keyv-3.1.1.tgz",
"integrity": "sha512-MPtoySlAZQ37VoLaPcTHCu1RWJ4llDkULYZIzOYxlhxBqYPB0RsRlmMU0R6tahtFe27mIdkHV+551ZWV4PLmVw==",
"requires": {
"@types/node": "*"
}
},
"@types/lodash": {
"version": "4.14.164",
"resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.164.tgz",
@@ -798,6 +827,14 @@
"form-data": "^2.5.0"
}
},
"@types/responselike": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/@types/responselike/-/responselike-1.0.0.tgz",
"integrity": "sha512-85Y2BjiufFzaMIlvJDvTTB8Fxl2xfLo4HgmHzVBz08w4wDePCTjYw66PdrolO0kzli3yam/YCgRufyo1DdQVTA==",
"requires": {
"@types/node": "*"
}
},
"@types/retry": {
"version": "0.12.0",
"resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz",
@@ -1927,6 +1964,11 @@
"unset-value": "^1.0.0"
}
},
"cacheable-lookup": {
"version": "5.0.3",
"resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-5.0.3.tgz",
"integrity": "sha512-W+JBqF9SWe18A72XFzN/V/CULFzPm7sBXzzR6ekkE+3tLG72wFZrBiBZhrZuDoYexop4PHJVdFAKb/Nj9+tm9w=="
},
"cacheable-request": {
"version": "6.1.0",
"resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-6.1.0.tgz",
@@ -2198,7 +2240,6 @@
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.2.tgz",
"integrity": "sha1-0dyXOSAxTfZ/vrlCI7TuNQI56Ws=",
"dev": true,
"requires": {
"mimic-response": "^1.0.0"
}
@@ -4696,8 +4737,40 @@
"http-cache-semantics": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz",
"integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ==",
"dev": true
"integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ=="
},
"http-proxy-agent": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz",
"integrity": "sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg==",
"requires": {
"@tootallnate/once": "1",
"agent-base": "6",
"debug": "4"
},
"dependencies": {
"agent-base": {
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
"integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==",
"requires": {
"debug": "4"
}
},
"debug": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz",
"integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==",
"requires": {
"ms": "2.1.2"
}
},
"ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
}
}
},
"http-signature": {
"version": "1.2.0",
@@ -4709,6 +4782,22 @@
"sshpk": "^1.7.0"
}
},
"http2-wrapper": {
"version": "1.0.0-beta.5.2",
"resolved": "https://registry.npmjs.org/http2-wrapper/-/http2-wrapper-1.0.0-beta.5.2.tgz",
"integrity": "sha512-xYz9goEyBnC8XwXDTuC/MZ6t+MrKVQZOk4s7+PaDkwIsQd8IwqvM+0M6bA/2lvG8GHXcPdf+MejTUeO2LCPCeQ==",
"requires": {
"quick-lru": "^5.1.1",
"resolve-alpn": "^1.0.0"
},
"dependencies": {
"quick-lru": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz",
"integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA=="
}
}
},
"https-browserify": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/https-browserify/-/https-browserify-1.0.0.tgz",
@@ -4834,6 +4923,11 @@
"integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==",
"dev": true
},
"ip": {
"version": "1.1.5",
"resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz",
"integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo="
},
"ircv3": {
"version": "0.26.5",
"resolved": "https://registry.npmjs.org/ircv3/-/ircv3-0.26.5.tgz",
@@ -5893,8 +5987,7 @@
"mimic-response": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz",
"integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==",
"dev": true
"integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ=="
},
"min-indent": {
"version": "1.0.1",
@@ -6437,8 +6530,7 @@
"normalize-url": {
"version": "4.5.0",
"resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.0.tgz",
"integrity": "sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ==",
"dev": true
"integrity": "sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ=="
},
"npm": {
"version": "7.0.12",
@@ -8953,6 +9045,159 @@
}
}
},
"puppeteer-page-proxy": {
"version": "1.2.8",
"resolved": "https://registry.npmjs.org/puppeteer-page-proxy/-/puppeteer-page-proxy-1.2.8.tgz",
"integrity": "sha512-dbidfqSV67UiN0Yk5xjXHCMZ+iXpMicWUwHS1PtvM1pFUhyRWWtq+bz7J/DO4wJ8hYof7wWk7dVIhtQsQ/Xl/A==",
"requires": {
"got": "^11.5.1",
"http-proxy-agent": "^4.0.1",
"https-proxy-agent": "^5.0.0",
"socks-proxy-agent": "^5.0.0",
"tough-cookie": "^4.0.0"
},
"dependencies": {
"@sindresorhus/is": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.0.0.tgz",
"integrity": "sha512-FyD2meJpDPjyNQejSjvnhpgI/azsQkA4lGbuu5BQZfjvJ9cbRZXzeWL2HceCekW4lixO9JPesIIQkSoLjeJHNQ=="
},
"@szmarczak/http-timer": {
"version": "4.0.5",
"resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-4.0.5.tgz",
"integrity": "sha512-PyRA9sm1Yayuj5OIoJ1hGt2YISX45w9WcFbh6ddT0Z/0yaFxOtGLInr4jUfU1EAFVs0Yfyfev4RNwBlUaHdlDQ==",
"requires": {
"defer-to-connect": "^2.0.0"
}
},
"agent-base": {
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
"integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==",
"requires": {
"debug": "4"
}
},
"cacheable-request": {
"version": "7.0.1",
"resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-7.0.1.tgz",
"integrity": "sha512-lt0mJ6YAnsrBErpTMWeu5kl/tg9xMAWjavYTN6VQXM1A/teBITuNcccXsCxF0tDQQJf9DfAaX5O4e0zp0KlfZw==",
"requires": {
"clone-response": "^1.0.2",
"get-stream": "^5.1.0",
"http-cache-semantics": "^4.0.0",
"keyv": "^4.0.0",
"lowercase-keys": "^2.0.0",
"normalize-url": "^4.1.0",
"responselike": "^2.0.0"
}
},
"debug": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz",
"integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==",
"requires": {
"ms": "2.1.2"
}
},
"decompress-response": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz",
"integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==",
"requires": {
"mimic-response": "^3.1.0"
}
},
"defer-to-connect": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.0.tgz",
"integrity": "sha512-bYL2d05vOSf1JEZNx5vSAtPuBMkX8K9EUutg7zlKvTqKXHt7RhWJFbmd7qakVuf13i+IkGmp6FwSsONOf6VYIg=="
},
"got": {
"version": "11.8.0",
"resolved": "https://registry.npmjs.org/got/-/got-11.8.0.tgz",
"integrity": "sha512-k9noyoIIY9EejuhaBNLyZ31D5328LeqnyPNXJQb2XlJZcKakLqN5m6O/ikhq/0lw56kUYS54fVm+D1x57YC9oQ==",
"requires": {
"@sindresorhus/is": "^4.0.0",
"@szmarczak/http-timer": "^4.0.5",
"@types/cacheable-request": "^6.0.1",
"@types/responselike": "^1.0.0",
"cacheable-lookup": "^5.0.3",
"cacheable-request": "^7.0.1",
"decompress-response": "^6.0.0",
"http2-wrapper": "^1.0.0-beta.5.2",
"lowercase-keys": "^2.0.0",
"p-cancelable": "^2.0.0",
"responselike": "^2.0.0"
}
},
"https-proxy-agent": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz",
"integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==",
"requires": {
"agent-base": "6",
"debug": "4"
}
},
"json-buffer": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz",
"integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ=="
},
"keyv": {
"version": "4.0.3",
"resolved": "https://registry.npmjs.org/keyv/-/keyv-4.0.3.tgz",
"integrity": "sha512-zdGa2TOpSZPq5mU6iowDARnMBZgtCqJ11dJROFi6tg6kTn4nuUdU09lFyLFSaHrWqpIJ+EBq4E8/Dc0Vx5vLdA==",
"requires": {
"json-buffer": "3.0.1"
}
},
"lowercase-keys": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz",
"integrity": "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA=="
},
"mimic-response": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz",
"integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ=="
},
"ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
"p-cancelable": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-2.0.0.tgz",
"integrity": "sha512-wvPXDmbMmu2ksjkB4Z3nZWTSkJEb9lqVdMaCKpZUGJG9TMiNp9XcbG3fn9fPKjem04fJMJnXoyFPk2FmgiaiNg=="
},
"responselike": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/responselike/-/responselike-2.0.0.tgz",
"integrity": "sha512-xH48u3FTB9VsZw7R+vvgaKeLKzT6jOogbQhEe/jewwnZgzPcnyWui2Av6JpoYZF/91uueC+lqhWqeURw5/qhCw==",
"requires": {
"lowercase-keys": "^2.0.0"
}
},
"tough-cookie": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.0.0.tgz",
"integrity": "sha512-tHdtEpQCMrc1YLrMaqXXcj6AxhYi/xgit6mZu1+EDWUn+qhUf8wMQoFIy9NXuq23zAwtcB0t/MjACGR18pcRbg==",
"requires": {
"psl": "^1.1.33",
"punycode": "^2.1.1",
"universalify": "^0.1.2"
}
},
"universalify": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz",
"integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg=="
}
}
},
"pushover-notifications": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/pushover-notifications/-/pushover-notifications-1.2.2.tgz",
@@ -9313,6 +9558,11 @@
"path-parse": "^1.0.6"
}
},
"resolve-alpn": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/resolve-alpn/-/resolve-alpn-1.0.0.tgz",
"integrity": "sha512-rTuiIEqFmGxne4IovivKSDzld2lWW9QCjqv80SYjPgf+gS35eaCAjaP54CCwGAwBtnCsvNLYtqxe1Nw+i6JEmA=="
},
"resolve-cwd": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz",
@@ -9630,6 +9880,11 @@
}
}
},
"smart-buffer": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.1.0.tgz",
"integrity": "sha512-iVICrxOzCynf/SNaBQCw34eM9jROU/s5rzIhpOvzhzuYHfJR/DhZfDkXiZSgKXfgv26HT3Yni3AV/DGw0cGnnw=="
},
"snake-case": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/snake-case/-/snake-case-3.0.3.tgz",
@@ -9756,6 +10011,48 @@
"kind-of": "^3.2.0"
}
},
"socks": {
"version": "2.5.0",
"resolved": "https://registry.npmjs.org/socks/-/socks-2.5.0.tgz",
"integrity": "sha512-00OqQHp5SCbwm9ecOMJj9aQtMSjwi1uVuGQoxnpKCS50VKZcOZ8z11CTKypmR8sEy7nZimy/qXY7rYJYbRlXmA==",
"requires": {
"ip": "^1.1.5",
"smart-buffer": "^4.1.0"
}
},
"socks-proxy-agent": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-5.0.0.tgz",
"integrity": "sha512-lEpa1zsWCChxiynk+lCycKuC502RxDWLKJZoIhnxrWNjLSDGYRFflHA1/228VkRcnv9TIb8w98derGbpKxJRgA==",
"requires": {
"agent-base": "6",
"debug": "4",
"socks": "^2.3.3"
},
"dependencies": {
"agent-base": {
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
"integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==",
"requires": {
"debug": "4"
}
},
"debug": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz",
"integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==",
"requires": {
"ms": "2.1.2"
}
},
"ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
}
}
},
"source-list-map": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/source-list-map/-/source-list-map-2.0.1.tgz",
package.json
+1
View File
@@ -45,6 +45,7 @@
"puppeteer-extra-plugin-adblocker": "^2.11.9",
"puppeteer-extra-plugin-block-resources": "^2.2.7",
"puppeteer-extra-plugin-stealth": "^2.6.5",
"puppeteer-page-proxy": "^1.2.8",
"pushover-notifications": "^1.2.2",
"twilio": "^3.52.0",
"twitch": "^4.3.2",
src/adblocker.ts
+13 -2
View File
@@ -5,9 +5,20 @@ export const adBlocker = new PuppeteerExtraPluginAdblocker({
blockTrackers: true
});
export async function disableBlockerInPage(page: Page) {
export async function enableBlockerInPage(page: Page) {
const blockerObject = await adBlocker.getBlocker();
if (blockerObject.isBlockingEnabled(page)) {
await blockerObject.disableBlockingInPage(page);
return;
}
await blockerObject.enableBlockingInPage(page);
}
export async function disableBlockerInPage(page: Page) {
const blockerObject = await adBlocker.getBlocker();
if (!blockerObject.isBlockingEnabled(page)) {
return;
}
await blockerObject.disableBlockingInPage(page);
}
src/config.ts
+13 -1
View File
@@ -2,6 +2,7 @@ import {banner} from './banner';
import {config as config_} from 'dotenv';
import path from 'path';
import {readFileSync} from 'fs';
config_({path: path.resolve(__dirname, '../.env')});
@@ -354,6 +355,16 @@ const store = {
]),
stores: envOrArray(process.env.STORES, ['nvidia']).map((entry) => {
const [name, minPageSleep, maxPageSleep] = entry.match(/[^:]+/g) ?? [];
let proxyList;
try {
proxyList = readFileSync(`${name}.proxies`)
.toString()
.trim()
.split('\n')
.map((x) => x.trim());
} catch {}
return {
maxPageSleep: envOrNumberMax(
minPageSleep,
@@ -365,7 +376,8 @@ const store = {
maxPageSleep,
browser.minSleep
),
name: envOrString(name)
name: envOrString(name),
proxyList
};
})
};
src/index.ts
-11
View File
@@ -1,25 +1,14 @@
import {startAPIServer, stopAPIServer} from './web';
import {Browser} from 'puppeteer';
import {adBlocker} from './adblocker';
import {config} from './config';
import {getSleepTime} from './util';
import {logger} from './logger';
import puppeteer from 'puppeteer-extra';
import resourceBlock from 'puppeteer-extra-plugin-block-resources';
import stealthPlugin from 'puppeteer-extra-plugin-stealth';
import {storeList} from './store/model';
import {tryLookupAndLoop} from './store';
puppeteer.use(stealthPlugin());
if (config.browser.lowBandwidth) {
puppeteer.use(
resourceBlock({
blockedTypes: new Set(['image', 'font'] as const)
})
);
} else {
puppeteer.use(adBlocker);
}
let browser: Browser | undefined;
src/store/lookup.ts
+162 -27
View File
@@ -1,4 +1,4 @@
import {Browser, Page, Response} from 'puppeteer';
import {Browser, Page, PageEventObj, Request, Response} from 'puppeteer';
import {Link, Store, getStores} from './model';
import {Print, logger} from '../logger';
import {Selector, cardPrice, pageIncludesLabels} from './includes-labels';
@@ -9,18 +9,109 @@ import {
getSleepTime,
isStatusCodeInRange
} from '../util';
import {disableBlockerInPage, enableBlockerInPage} from '../adblocker';
import {config} from '../config';
import {disableBlockerInPage} from '../adblocker';
import {fetchLinks} from './fetch-links';
import {filterStoreLink} from './filter';
import open from 'open';
import {processBackoffDelay} from './model/helpers/backoff';
import {sendNotification} from '../notification';
import useProxy from 'puppeteer-page-proxy';
const inStock: Record<string, boolean> = {};
const linkBuilderLastRunTimes: Record<string, number> = {};
function nextProxy(store: Store) {
if (!store.proxyList) {
return;
}
if (store.currentProxyIndex === undefined) {
store.currentProxyIndex = 0;
}
store.currentProxyIndex++;
if (store.currentProxyIndex >= store.proxyList.length) {
store.currentProxyIndex = 0;
}
logger.info(
`ℹ [${store.name}] Next proxy index: ${store.currentProxyIndex} / Count: ${store.proxyList.length}`
);
return store.proxyList[store.currentProxyIndex];
}
async function handleLowBandwidth(request: Request) {
if (!config.browser.lowBandwidth) {
return false;
}
const typ = request.resourceType();
if (typ === 'font' || typ === 'image') {
try {
await request.abort();
} catch {}
return true;
}
return false;
}
async function handleProxy(request: Request, proxy?: string) {
if (!proxy) {
return false;
}
try {
await useProxy(request, proxy);
} catch (error: unknown) {
logger.error(error);
try {
await request.abort();
} catch {}
}
return true;
}
async function handleAdBlock(request: Request, adBlockRequestHandler: any) {
if (!adBlockRequestHandler) {
return false;
}
return new Promise((resolve) => {
const continueFunc = async () => {
resolve(false);
};
const abortFunc = async () => {
try {
await request.abort();
} catch {}
resolve(true);
};
const requestProxy = new Proxy(request, {
get(target, prop, receiver) {
if (prop === 'continue') {
return continueFunc;
}
if (prop === 'abort') {
return abortFunc;
}
return Reflect.get(target, prop, receiver);
}
});
adBlockRequestHandler(requestProxy);
});
}
/**
* Responsible for looking up information about a each product within
* a `Store`. It's important that we ignore `no-await-in-loop` here
@@ -34,6 +125,20 @@ async function lookup(browser: Browser, store: Store) {
return;
}
if (store.linksBuilder) {
logger.info(`[${store.name}] Running linksBuilder...`);
const lastRunTime = linkBuilderLastRunTimes[store.name] ?? -1;
const ttl = store.linksBuilder.ttl ?? Number.MAX_SAFE_INTEGER;
if (lastRunTime === -1 || Date.now() - lastRunTime > ttl) {
try {
await fetchLinks(store, browser);
linkBuilderLastRunTimes[store.name] = Date.now();
} catch (error: unknown) {
logger.error(error);
}
}
}
/* eslint-disable no-await-in-loop */
for (const link of store.links) {
if (!filterStoreLink(link)) {
@@ -45,23 +150,62 @@ async function lookup(browser: Browser, store: Store) {
continue;
}
const context = config.browser.isIncognito
const proxy = nextProxy(store);
const useAdBlock = !config.browser.lowBandwidth && !store.disableAdBlocker;
const customContext = config.browser.isIncognito;
const context = customContext
? await browser.createIncognitoBrowserContext()
: browser.defaultBrowserContext();
const page = config.browser.isIncognito
? await context.newPage()
: await browser.newPage();
const page = await context.newPage();
page.setDefaultNavigationTimeout(config.page.timeout);
await page.setUserAgent(getRandomUserAgent());
if (store.disableAdBlocker) {
try {
await disableBlockerInPage(page);
} catch (error: unknown) {
logger.error(error);
}
let adBlockRequestHandler: any;
let pageProxy;
if (useAdBlock) {
const onProxyFunc = (event: keyof PageEventObj, handler: any) => {
if (event !== 'request') {
page.on(event, handler);
return;
}
adBlockRequestHandler = handler;
};
pageProxy = new Proxy(page, {
get(target, prop, receiver) {
if (prop === 'on') {
return onProxyFunc;
}
return Reflect.get(target, prop, receiver);
}
});
await enableBlockerInPage(pageProxy);
}
await page.setRequestInterception(true);
page.on('request', async (request) => {
if (await handleLowBandwidth(request)) {
return;
}
if (await handleAdBlock(request, adBlockRequestHandler)) {
return;
}
if (await handleProxy(request, proxy)) {
return;
}
try {
await request.continue();
} catch {}
});
let statusCode = 0;
try {
@@ -74,7 +218,11 @@ async function lookup(browser: Browser, store: Store) {
);
const client = await page.target().createCDPSession();
await client.send('Network.clearBrowserCookies');
await client.send('Network.clearBrowserCache');
// Await client.send('Network.clearBrowserCache');
}
if (pageProxy) {
await disableBlockerInPage(pageProxy);
}
// Must apply backoff before closing the page, e.g. if CloudFlare is
@@ -82,7 +230,7 @@ async function lookup(browser: Browser, store: Store) {
// before redirecting to the next page
await processBackoffDelay(store, link, statusCode);
await closePage(page);
if (config.browser.isIncognito) {
if (customContext) {
await context.close();
}
}
@@ -223,19 +371,6 @@ export async function tryLookupAndLoop(browser: Browser, store: Store) {
return;
}
if (getStores().has(store.name) && store.linksBuilder) {
const lastRunTime = linkBuilderLastRunTimes[store.name] ?? -1;
const ttl = store.linksBuilder.ttl ?? Number.MAX_SAFE_INTEGER;
if (lastRunTime === -1 || Date.now() - lastRunTime > ttl) {
try {
await fetchLinks(store, browser);
linkBuilderLastRunTimes[store.name] = Date.now();
} catch (error: unknown) {
logger.error((error as Error).message);
}
}
}
logger.debug(`[${store.name}] Starting lookup...`);
try {
await lookup(browser, store);
src/store/model/index.ts
+1
View File
@@ -236,6 +236,7 @@ export function updateStores() {
stores.set(storeData.name, store);
store.minPageSleep = storeData.minPageSleep;
store.maxPageSleep = storeData.maxPageSleep;
store.proxyList = storeData.proxyList;
} else {
logger.warn(`No store named ${storeData.name}, skipping.`);
}
src/store/model/store.ts
+4 -1
View File
@@ -1,4 +1,4 @@
import {Browser, LoadEvent} from 'puppeteer';
import {Browser, BrowserContext, LoadEvent} from 'puppeteer';
export type Element = {
container?: string;
@@ -178,4 +178,7 @@ export type Store = {
waitUntil?: LoadEvent;
minPageSleep?: number;
maxPageSleep?: number;
proxyList?: string[];
currentProxyIndex?: number;
};