fix(proxy): fix requests with proxies (#1408)

This commit is contained in:
Mark Dietzer
2020-12-15 14:08:38 -08:00
committed by GitHub
parent ecdb7a058d
commit f65df4ce56
5 changed files with 181 additions and 170 deletions
+1
View File
@@ -7,6 +7,7 @@ src/config/*.yaml
.env
dotenv
*.proxies
success-*.png
*.wav
+165 -165
View File
@@ -410,6 +410,159 @@
"mime-types": "^2.1.12"
}
},
"@doridian/puppeteer-page-proxy": {
"version": "1.2.10",
"resolved": "https://registry.npmjs.org/@doridian/puppeteer-page-proxy/-/puppeteer-page-proxy-1.2.10.tgz",
"integrity": "sha512-+Oc2JYaI00gaPtUfgMrCLpqBtr5MmzSWny0LYrvFqTsXUUMJqvkB3LJdVGeo24UzyB+TPJV7aQOJkZVEPeUP3A==",
"requires": {
"got": "^11.5.1",
"http-proxy-agent": "^4.0.1",
"https-proxy-agent": "^5.0.0",
"socks-proxy-agent": "^5.0.0",
"tough-cookie": "^4.0.0"
},
"dependencies": {
"@sindresorhus/is": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.0.0.tgz",
"integrity": "sha512-FyD2meJpDPjyNQejSjvnhpgI/azsQkA4lGbuu5BQZfjvJ9cbRZXzeWL2HceCekW4lixO9JPesIIQkSoLjeJHNQ=="
},
"@szmarczak/http-timer": {
"version": "4.0.5",
"resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-4.0.5.tgz",
"integrity": "sha512-PyRA9sm1Yayuj5OIoJ1hGt2YISX45w9WcFbh6ddT0Z/0yaFxOtGLInr4jUfU1EAFVs0Yfyfev4RNwBlUaHdlDQ==",
"requires": {
"defer-to-connect": "^2.0.0"
}
},
"agent-base": {
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
"integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==",
"requires": {
"debug": "4"
}
},
"cacheable-request": {
"version": "7.0.1",
"resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-7.0.1.tgz",
"integrity": "sha512-lt0mJ6YAnsrBErpTMWeu5kl/tg9xMAWjavYTN6VQXM1A/teBITuNcccXsCxF0tDQQJf9DfAaX5O4e0zp0KlfZw==",
"requires": {
"clone-response": "^1.0.2",
"get-stream": "^5.1.0",
"http-cache-semantics": "^4.0.0",
"keyv": "^4.0.0",
"lowercase-keys": "^2.0.0",
"normalize-url": "^4.1.0",
"responselike": "^2.0.0"
}
},
"debug": {
"version": "4.3.1",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz",
"integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==",
"requires": {
"ms": "2.1.2"
}
},
"decompress-response": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz",
"integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==",
"requires": {
"mimic-response": "^3.1.0"
}
},
"defer-to-connect": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.0.tgz",
"integrity": "sha512-bYL2d05vOSf1JEZNx5vSAtPuBMkX8K9EUutg7zlKvTqKXHt7RhWJFbmd7qakVuf13i+IkGmp6FwSsONOf6VYIg=="
},
"got": {
"version": "11.8.1",
"resolved": "https://registry.npmjs.org/got/-/got-11.8.1.tgz",
"integrity": "sha512-9aYdZL+6nHmvJwHALLwKSUZ0hMwGaJGYv3hoPLPgnT8BoBXm1SjnZeky+91tfwJaDzun2s4RsBRy48IEYv2q2Q==",
"requires": {
"@sindresorhus/is": "^4.0.0",
"@szmarczak/http-timer": "^4.0.5",
"@types/cacheable-request": "^6.0.1",
"@types/responselike": "^1.0.0",
"cacheable-lookup": "^5.0.3",
"cacheable-request": "^7.0.1",
"decompress-response": "^6.0.0",
"http2-wrapper": "^1.0.0-beta.5.2",
"lowercase-keys": "^2.0.0",
"p-cancelable": "^2.0.0",
"responselike": "^2.0.0"
}
},
"https-proxy-agent": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz",
"integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==",
"requires": {
"agent-base": "6",
"debug": "4"
}
},
"json-buffer": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz",
"integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ=="
},
"keyv": {
"version": "4.0.3",
"resolved": "https://registry.npmjs.org/keyv/-/keyv-4.0.3.tgz",
"integrity": "sha512-zdGa2TOpSZPq5mU6iowDARnMBZgtCqJ11dJROFi6tg6kTn4nuUdU09lFyLFSaHrWqpIJ+EBq4E8/Dc0Vx5vLdA==",
"requires": {
"json-buffer": "3.0.1"
}
},
"lowercase-keys": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz",
"integrity": "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA=="
},
"mimic-response": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz",
"integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ=="
},
"ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
"p-cancelable": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-2.0.0.tgz",
"integrity": "sha512-wvPXDmbMmu2ksjkB4Z3nZWTSkJEb9lqVdMaCKpZUGJG9TMiNp9XcbG3fn9fPKjem04fJMJnXoyFPk2FmgiaiNg=="
},
"responselike": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/responselike/-/responselike-2.0.0.tgz",
"integrity": "sha512-xH48u3FTB9VsZw7R+vvgaKeLKzT6jOogbQhEe/jewwnZgzPcnyWui2Av6JpoYZF/91uueC+lqhWqeURw5/qhCw==",
"requires": {
"lowercase-keys": "^2.0.0"
}
},
"tough-cookie": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.0.0.tgz",
"integrity": "sha512-tHdtEpQCMrc1YLrMaqXXcj6AxhYi/xgit6mZu1+EDWUn+qhUf8wMQoFIy9NXuq23zAwtcB0t/MjACGR18pcRbg==",
"requires": {
"psl": "^1.1.33",
"punycode": "^2.1.1",
"universalify": "^0.1.2"
}
},
"universalify": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz",
"integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg=="
}
}
},
"@eslint/eslintrc": {
"version": "0.2.2",
"resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.2.2.tgz",
@@ -2008,9 +2161,9 @@
}
},
"cacheable-lookup": {
"version": "5.0.3",
"resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-5.0.3.tgz",
"integrity": "sha512-W+JBqF9SWe18A72XFzN/V/CULFzPm7sBXzzR6ekkE+3tLG72wFZrBiBZhrZuDoYexop4PHJVdFAKb/Nj9+tm9w=="
"version": "5.0.4",
"resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-5.0.4.tgz",
"integrity": "sha512-2/kNscPhpcxrOigMZzbiWF7dz8ilhb/nIHU3EyZiXWXpeq/au8qJ8VhdftMkty3n7Gj6HIGalQG8oiBNB3AJgA=="
},
"cacheable-request": {
"version": "6.1.0",
@@ -4775,9 +4928,9 @@
}
},
"debug": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz",
"integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==",
"version": "4.3.1",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz",
"integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==",
"requires": {
"ms": "2.1.2"
}
@@ -9079,159 +9232,6 @@
}
}
},
"puppeteer-page-proxy": {
"version": "1.2.8",
"resolved": "https://registry.npmjs.org/puppeteer-page-proxy/-/puppeteer-page-proxy-1.2.8.tgz",
"integrity": "sha512-dbidfqSV67UiN0Yk5xjXHCMZ+iXpMicWUwHS1PtvM1pFUhyRWWtq+bz7J/DO4wJ8hYof7wWk7dVIhtQsQ/Xl/A==",
"requires": {
"got": "^11.5.1",
"http-proxy-agent": "^4.0.1",
"https-proxy-agent": "^5.0.0",
"socks-proxy-agent": "^5.0.0",
"tough-cookie": "^4.0.0"
},
"dependencies": {
"@sindresorhus/is": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.0.0.tgz",
"integrity": "sha512-FyD2meJpDPjyNQejSjvnhpgI/azsQkA4lGbuu5BQZfjvJ9cbRZXzeWL2HceCekW4lixO9JPesIIQkSoLjeJHNQ=="
},
"@szmarczak/http-timer": {
"version": "4.0.5",
"resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-4.0.5.tgz",
"integrity": "sha512-PyRA9sm1Yayuj5OIoJ1hGt2YISX45w9WcFbh6ddT0Z/0yaFxOtGLInr4jUfU1EAFVs0Yfyfev4RNwBlUaHdlDQ==",
"requires": {
"defer-to-connect": "^2.0.0"
}
},
"agent-base": {
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
"integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==",
"requires": {
"debug": "4"
}
},
"cacheable-request": {
"version": "7.0.1",
"resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-7.0.1.tgz",
"integrity": "sha512-lt0mJ6YAnsrBErpTMWeu5kl/tg9xMAWjavYTN6VQXM1A/teBITuNcccXsCxF0tDQQJf9DfAaX5O4e0zp0KlfZw==",
"requires": {
"clone-response": "^1.0.2",
"get-stream": "^5.1.0",
"http-cache-semantics": "^4.0.0",
"keyv": "^4.0.0",
"lowercase-keys": "^2.0.0",
"normalize-url": "^4.1.0",
"responselike": "^2.0.0"
}
},
"debug": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz",
"integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==",
"requires": {
"ms": "2.1.2"
}
},
"decompress-response": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz",
"integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==",
"requires": {
"mimic-response": "^3.1.0"
}
},
"defer-to-connect": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.0.tgz",
"integrity": "sha512-bYL2d05vOSf1JEZNx5vSAtPuBMkX8K9EUutg7zlKvTqKXHt7RhWJFbmd7qakVuf13i+IkGmp6FwSsONOf6VYIg=="
},
"got": {
"version": "11.8.0",
"resolved": "https://registry.npmjs.org/got/-/got-11.8.0.tgz",
"integrity": "sha512-k9noyoIIY9EejuhaBNLyZ31D5328LeqnyPNXJQb2XlJZcKakLqN5m6O/ikhq/0lw56kUYS54fVm+D1x57YC9oQ==",
"requires": {
"@sindresorhus/is": "^4.0.0",
"@szmarczak/http-timer": "^4.0.5",
"@types/cacheable-request": "^6.0.1",
"@types/responselike": "^1.0.0",
"cacheable-lookup": "^5.0.3",
"cacheable-request": "^7.0.1",
"decompress-response": "^6.0.0",
"http2-wrapper": "^1.0.0-beta.5.2",
"lowercase-keys": "^2.0.0",
"p-cancelable": "^2.0.0",
"responselike": "^2.0.0"
}
},
"https-proxy-agent": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz",
"integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==",
"requires": {
"agent-base": "6",
"debug": "4"
}
},
"json-buffer": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz",
"integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ=="
},
"keyv": {
"version": "4.0.3",
"resolved": "https://registry.npmjs.org/keyv/-/keyv-4.0.3.tgz",
"integrity": "sha512-zdGa2TOpSZPq5mU6iowDARnMBZgtCqJ11dJROFi6tg6kTn4nuUdU09lFyLFSaHrWqpIJ+EBq4E8/Dc0Vx5vLdA==",
"requires": {
"json-buffer": "3.0.1"
}
},
"lowercase-keys": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz",
"integrity": "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA=="
},
"mimic-response": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz",
"integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ=="
},
"ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
"p-cancelable": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-2.0.0.tgz",
"integrity": "sha512-wvPXDmbMmu2ksjkB4Z3nZWTSkJEb9lqVdMaCKpZUGJG9TMiNp9XcbG3fn9fPKjem04fJMJnXoyFPk2FmgiaiNg=="
},
"responselike": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/responselike/-/responselike-2.0.0.tgz",
"integrity": "sha512-xH48u3FTB9VsZw7R+vvgaKeLKzT6jOogbQhEe/jewwnZgzPcnyWui2Av6JpoYZF/91uueC+lqhWqeURw5/qhCw==",
"requires": {
"lowercase-keys": "^2.0.0"
}
},
"tough-cookie": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.0.0.tgz",
"integrity": "sha512-tHdtEpQCMrc1YLrMaqXXcj6AxhYi/xgit6mZu1+EDWUn+qhUf8wMQoFIy9NXuq23zAwtcB0t/MjACGR18pcRbg==",
"requires": {
"psl": "^1.1.33",
"punycode": "^2.1.1",
"universalify": "^0.1.2"
}
},
"universalify": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz",
"integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg=="
}
}
},
"pushover-notifications": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/pushover-notifications/-/pushover-notifications-1.2.2.tgz",
@@ -10049,9 +10049,9 @@
}
},
"socks": {
"version": "2.5.0",
"resolved": "https://registry.npmjs.org/socks/-/socks-2.5.0.tgz",
"integrity": "sha512-00OqQHp5SCbwm9ecOMJj9aQtMSjwi1uVuGQoxnpKCS50VKZcOZ8z11CTKypmR8sEy7nZimy/qXY7rYJYbRlXmA==",
"version": "2.5.1",
"resolved": "https://registry.npmjs.org/socks/-/socks-2.5.1.tgz",
"integrity": "sha512-oZCsJJxapULAYJaEYBSzMcz8m3jqgGrHaGhkmU/o/PQfFWYWxkAaA0UMGImb6s6tEXfKi959X6VJjMMQ3P6TTQ==",
"requires": {
"ip": "^1.1.5",
"smart-buffer": "^4.1.0"
@@ -10076,9 +10076,9 @@
}
},
"debug": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.2.0.tgz",
"integrity": "sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==",
"version": "4.3.1",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz",
"integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==",
"requires": {
"ms": "2.1.2"
}
+1 -1
View File
@@ -25,6 +25,7 @@
},
"homepage": "https://github.com/jef/streetmerchant#readme",
"dependencies": {
"@doridian/puppeteer-page-proxy": "^1.2.10",
"@jef/pushbullet": "^2.4.3",
"@slack/web-api": "^5.14.0",
"@types/random-useragent": "^0.3.0",
@@ -46,7 +47,6 @@
"puppeteer-extra-plugin-adblocker": "^2.11.9",
"puppeteer-extra-plugin-block-resources": "^2.2.7",
"puppeteer-extra-plugin-stealth": "^2.6.5",
"puppeteer-page-proxy": "^1.2.8",
"pushover-notifications": "^1.2.2",
"random-useragent": "^0.5.0",
"twilio": "^3.54.0",
+9 -3
View File
@@ -14,7 +14,8 @@ import {
delay,
getRandomUserAgent,
getSleepTime,
isStatusCodeInRange
isStatusCodeInRange,
noop
} from '../util';
import {disableBlockerInPage, enableBlockerInPage} from '../adblocker';
import {config} from '../config';
@@ -23,7 +24,7 @@ import {filterStoreLink} from './filter';
import open from 'open';
import {processBackoffDelay} from './model/helpers/backoff';
import {sendNotification} from '../notification';
import useProxy from 'puppeteer-page-proxy';
import useProxy from '@doridian/puppeteer-page-proxy';
const inStock: Record<string, boolean> = {};
@@ -75,7 +76,7 @@ async function handleProxy(request: Request, proxy?: string) {
try {
await useProxy(request, proxy);
} catch (error: unknown) {
logger.error(error);
logger.error('handleProxy', error);
try {
await request.abort();
} catch {}
@@ -203,6 +204,11 @@ async function lookup(browser: Browser, store: Store) {
return onProxyFunc;
}
// Give dummy setRequestInterception to avoid AdBlock from messing with it
if (prop === 'setRequestInterception') {
return noop;
}
return Reflect.get(target, prop, receiver);
}
});
+5 -1
View File
@@ -18,6 +18,10 @@ export async function delay(ms: number) {
});
}
export function noop() {
// Do nothing
}
export function isStatusCodeInRange(
statusCode: number,
range: StatusCodeRangeArray
@@ -66,7 +70,7 @@ export async function usingPage<T>(
try {
await closePage(page);
} catch (error: unknown) {
logger.error(error);
logger.error('usingPage', error);
}
}
}