opelly/streetmerchant
1
0
Fork 0
mirror of https://github.com/opelly27/streetmerchant.git synced 2026-05-20 06:27:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: load puppeteer faster, run stores in parallel (#83)

Browse Source 
Co-authored-by: Jef LeCompte <jeffreylec@gmail.com>
This commit is contained in:
Jordan Garcia
2020-09-20 10:28:45 -04:00
committed by GitHub
parent a501cf703b
commit d1a5aa1f02
9 changed files with 349 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package-lock.json
Generated
+280 -12
View File
@@ -123,6 +123,45 @@
"to-fast-properties": "^2.0.0"
}
},
"@cliqz/adblocker": {
"version": "1.18.3",
"resolved": "https://registry.npmjs.org/@cliqz/adblocker/-/adblocker-1.18.3.tgz",
"integrity": "sha512-fkGky+ffAsXw9WIS+cV9zm8EMzdjRKU/uO196yCFHYICByZyREBie3lMNNKQ6RVSUeEVFOx1JlEKkY9Bze/9xQ==",
"requires": {
"@remusao/guess-url-type": "^1.1.2",
"@remusao/small": "^1.1.2",
"@remusao/smaz": "^1.7.1",
"@types/chrome": "^0.0.123",
"@types/firefox-webext-browser": "^78.0.0",
"tldts-experimental": "^5.6.21"
},
"dependencies": {
"@types/chrome": {
"version": "0.0.123",
"resolved": "https://registry.npmjs.org/@types/chrome/-/chrome-0.0.123.tgz",
"integrity": "sha512-fG6GPreuSY+Z+0e3dtBz5MJ5qyZ2feOZISG8udxBiuwUYqykK1q4NxkjfzL2F5I05LqK2ojP7ZR08Gcfo3ubHQ==",
"requires": {
"@types/filesystem": "*",
"@types/har-format": "*"
}
}
}
},
"@cliqz/adblocker-content": {
"version": "1.18.3",
"resolved": "https://registry.npmjs.org/@cliqz/adblocker-content/-/adblocker-content-1.18.3.tgz",
"integrity": "sha512-mCLlGg4B8P2VWtJpSAJStR9HeRNt5Jo4D0MIOdXIkdSFjCWcXUSwqlUtu5GBvA8iFp9cGgHC/EYeyUW1SbuvYg=="
},
"@cliqz/adblocker-puppeteer": {
"version": "1.18.3",
"resolved": "https://registry.npmjs.org/@cliqz/adblocker-puppeteer/-/adblocker-puppeteer-1.18.3.tgz",
"integrity": "sha512-2JkMzGeC2+s2t7oZHZLiBt7b3RTyo5kC3Uewih9CapRo/3xWUugIqBvmik0Q+Pr1/DE3x7YB36GuyLwxbt1yZg==",
"requires": {
"@cliqz/adblocker": "^1.18.3",
"@cliqz/adblocker-content": "^1.18.3",
"tldts-experimental": "^5.6.21"
}
},
"@dabh/diagnostics": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.2.tgz",
@@ -167,6 +206,43 @@
"integrity": "sha512-shAmDyaQC4H92APFoIaVDHCx5bStIocgvbwQyxPRrbUY20V1EYTbSDchWbuwlMG3V17cprZhA6+78JfB+3DTPw==",
"dev": true
},
"@remusao/guess-url-type": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@remusao/guess-url-type/-/guess-url-type-1.2.0.tgz",
"integrity": "sha512-alnTonifD/Ii/0pI9EA5nVgdk/eOihU4OOYMIXq4U4cS0NocnaYCozqV4OVkmArPPnz9s4ap4GM1ODftBpBW0w=="
},
"@remusao/small": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@remusao/small/-/small-1.2.0.tgz",
"integrity": "sha512-18Bwa/EjqQ5WfdERqmG3YgOohO7J2sS8+v31JgmYnEg3wAtcAOPVBRkD24IzVS0eJOQk1P2Yd++aP0ldirk7MQ=="
},
"@remusao/smaz": {
"version": "1.9.0",
"resolved": "https://registry.npmjs.org/@remusao/smaz/-/smaz-1.9.0.tgz",
"integrity": "sha512-HMMPam5jLhP0ymtMUQ8sm2p9zwDJwHD09krORXN/l/TR+NlSCdU2gSAoVNr9idD9OmMGfeXPFQYCofEUZfjbTQ==",
"requires": {
"@remusao/smaz-compress": "^1.9.0",
"@remusao/smaz-decompress": "^1.9.0"
}
},
"@remusao/smaz-compress": {
"version": "1.9.0",
"resolved": "https://registry.npmjs.org/@remusao/smaz-compress/-/smaz-compress-1.9.0.tgz",
"integrity": "sha512-PAze3aYCcUfX+a6E6sVMoxVtUkeQgX+oiY6DqbiRkNtUqzjtcl9JVyEAWGbBEgOuv2jdEATAlyIf0W18NKDEnw==",
"requires": {
"@remusao/trie": "^1.4.0"
}
},
"@remusao/smaz-decompress": {
"version": "1.9.0",
"resolved": "https://registry.npmjs.org/@remusao/smaz-decompress/-/smaz-decompress-1.9.0.tgz",
"integrity": "sha512-7uXEX8cSMWy+ai7j8sJpVQuY+CHj2e5D+PjxY//4wbAJlw1a/X+CYPt7BuxLBzpVoioB5Y7++1USjCkrw0pl8g=="
},
"@remusao/trie": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/@remusao/trie/-/trie-1.4.0.tgz",
"integrity": "sha512-mIr0m4/xj6qxHtJjAFb4I8tXXmjTniUYTB2Hv+xK5hXf/YWocEPlJ+V31bv5HJwo6ly64DUnZDBeBxolT3WE7w=="
},
"@sindresorhus/is": {
"version": "0.14.0",
"resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.14.0.tgz",
@@ -224,6 +300,20 @@
"defer-to-connect": "^1.0.1"
}
},
"@types/async": {
"version": "3.2.3",
"resolved": "https://registry.npmjs.org/@types/async/-/async-3.2.3.tgz",
"integrity": "sha512-deXFjLZc1h6SOh3hicVgD+S2EAkhSBGX/vdlD4nTzCjjOFQ+bfNiXocQ21xJjFAUwqaCeyvOQMgrnbg4QEV63A==",
"dev": true
},
"@types/chrome": {
"version": "0.0.91",
"resolved": "https://registry.npmjs.org/@types/chrome/-/chrome-0.0.91.tgz",
"integrity": "sha512-vNvo9lJkp1AvViWrUwe1bxhoMwr5dRZWlgr1DTuaNkz97LsG56lDX1sceWeZir2gRACJ5vdHtoRdVAvm8C75Ug==",
"requires": {
"@types/filesystem": "*"
}
},
"@types/color-name": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/@types/color-name/-/color-name-1.1.1.tgz",
@@ -241,6 +331,24 @@
"integrity": "sha512-OCutwjDZ4aFS6PB1UZ988C4YgwlBHJd6wCeQqaLdmadZ/7e+w79+hbMUFC1QXDNCmdyoRfAFdm0RypzwR+Qpag==",
"dev": true
},
"@types/filesystem": {
"version": "0.0.29",
"resolved": "https://registry.npmjs.org/@types/filesystem/-/filesystem-0.0.29.tgz",
"integrity": "sha512-85/1KfRedmfPGsbK8YzeaQUyV1FQAvMPMTuWFQ5EkLd2w7szhNO96bk3Rh/SKmOfd9co2rCLf0Voy4o7ECBOvw==",
"requires": {
"@types/filewriter": "*"
}
},
"@types/filewriter": {
"version": "0.0.28",
"resolved": "https://registry.npmjs.org/@types/filewriter/-/filewriter-0.0.28.tgz",
"integrity": "sha1-wFTor02d11205jq8dviFFocU1LM="
},
"@types/firefox-webext-browser": {
"version": "78.0.1",
"resolved": "https://registry.npmjs.org/@types/firefox-webext-browser/-/firefox-webext-browser-78.0.1.tgz",
"integrity": "sha512-0d7oiI9K6Y4efP4Crl3JB88zYl7vaRdLtumqz8v6axMF8RCnK0NaGUjL4DnyQ7GLPo98b+s0BSRalaxAXgvPAQ=="
},
"@types/glob": {
"version": "7.1.3",
"resolved": "https://registry.npmjs.org/@types/glob/-/glob-7.1.3.tgz",
@@ -251,6 +359,11 @@
"@types/node": "*"
}
},
"@types/har-format": {
"version": "1.2.4",
"resolved": "https://registry.npmjs.org/@types/har-format/-/har-format-1.2.4.tgz",
"integrity": "sha512-iUxzm1meBm3stxUMzRqgOVHjj4Kgpgu5w9fm4X7kPRfSgVRzythsucEN7/jtOo8SQzm+HfcxWWzJS0mJDH/3DQ=="
},
"@types/is-stream": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@types/is-stream/-/is-stream-1.1.0.tgz",
@@ -325,7 +438,6 @@
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/@types/puppeteer/-/puppeteer-3.0.2.tgz",
"integrity": "sha512-JRuHPSbHZBadOxxFwpyZPeRlpPTTeMbQneMdpFd8LXdyNfFSiX950CGewdm69g/ipzEAXAmMyFF1WOWJOL/nKw==",
"dev": true,
"requires": {
"@types/node": "*"
}
@@ -526,8 +638,7 @@
"arr-union": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/arr-union/-/arr-union-3.1.0.tgz",
"integrity": "sha1-45sJrqne+Gao8gbiiK9jkZuuOcQ=",
"dev": true
"integrity": "sha1-45sJrqne+Gao8gbiiK9jkZuuOcQ="
},
"array-find": {
"version": "1.0.0",
@@ -1174,6 +1285,28 @@
"integrity": "sha512-y4coMcylgSCdVinjiDBuR8PCC2bLjyGTwEmPb9NHR/QaNU6EUOXcTY/s6VjGMD6ENSEaeQYHCY0GNGS5jfMwPw==",
"dev": true
},
"clone-deep": {
"version": "0.2.4",
"resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-0.2.4.tgz",
"integrity": "sha1-TnPdCen7lxzDhnDF3O2cGJZIHMY=",
"requires": {
"for-own": "^0.1.3",
"is-plain-object": "^2.0.1",
"kind-of": "^3.0.2",
"lazy-cache": "^1.0.3",
"shallow-clone": "^0.1.2"
},
"dependencies": {
"kind-of": {
"version": "3.2.2",
"resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
"integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
"requires": {
"is-buffer": "^1.1.5"
}
}
}
},
"clone-response": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.2.tgz",
@@ -1485,6 +1618,11 @@
"core-assert": "^0.2.0"
}
},
"deepmerge": {
"version": "4.2.2",
"resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.2.2.tgz",
"integrity": "sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg=="
},
"defer-to-connect": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-1.1.3.tgz",
@@ -2854,8 +2992,15 @@
"for-in": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz",
"integrity": "sha1-gQaNKVqBQuwKxybG4iAMMPttXoA=",
"dev": true
"integrity": "sha1-gQaNKVqBQuwKxybG4iAMMPttXoA="
},
"for-own": {
"version": "0.1.5",
"resolved": "https://registry.npmjs.org/for-own/-/for-own-0.1.5.tgz",
"integrity": "sha1-UmXGgaTylNq78XyVCbZ2OqhFEM4=",
"requires": {
"for-in": "^1.0.1"
}
},
"form-data": {
"version": "2.5.1",
@@ -3297,8 +3442,7 @@
"is-buffer": {
"version": "1.1.6",
"resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
"integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==",
"dev": true
"integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w=="
},
"is-callable": {
"version": "1.2.1",
@@ -3374,8 +3518,7 @@
"is-extendable": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz",
"integrity": "sha1-YrEQ4omkcUGOPsNqYX1HLjAd/Ik=",
"dev": true
"integrity": "sha1-YrEQ4omkcUGOPsNqYX1HLjAd/Ik="
},
"is-extglob": {
"version": "2.1.1",
@@ -3497,7 +3640,6 @@
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz",
"integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==",
"dev": true,
"requires": {
"isobject": "^3.0.1"
}
@@ -3599,8 +3741,7 @@
"isobject": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
"integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=",
"dev": true
"integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8="
},
"js-tokens": {
"version": "4.0.0",
@@ -3701,6 +3842,11 @@
"package-json": "^6.3.0"
}
},
"lazy-cache": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-1.0.4.tgz",
"integrity": "sha1-odePw6UEdMuAhF07O24dpJpEbo4="
},
"levn": {
"version": "0.4.1",
"resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz",
@@ -3991,6 +4137,26 @@
}
}
},
"merge-deep": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/merge-deep/-/merge-deep-3.0.2.tgz",
"integrity": "sha512-T7qC8kg4Zoti1cFd8Cr0M+qaZfOwjlPDEdZIIPPB2JZctjaPM4fX+i7HOId69tAti2fvO6X5ldfYUONDODsrkA==",
"requires": {
"arr-union": "^3.1.0",
"clone-deep": "^0.2.4",
"kind-of": "^3.0.2"
},
"dependencies": {
"kind-of": {
"version": "3.2.2",
"resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
"integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
"requires": {
"is-buffer": "^1.1.5"
}
}
}
},
"merge2": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz",
@@ -4197,6 +4363,22 @@
}
}
},
"mixin-object": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/mixin-object/-/mixin-object-2.0.1.tgz",
"integrity": "sha1-T7lJRB2rGCVA8f4DW6YOGUel5X4=",
"requires": {
"for-in": "^0.1.3",
"is-extendable": "^0.1.1"
},
"dependencies": {
"for-in": {
"version": "0.1.8",
"resolved": "https://registry.npmjs.org/for-in/-/for-in-0.1.8.tgz",
"integrity": "sha1-2Hc5COMSVhCZUrH9ubP6hn0ndeE="
}
}
},
"mkdirp": {
"version": "0.5.5",
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz",
@@ -4256,6 +4438,11 @@
"tslib": "^1.10.0"
}
},
"node-fetch": {
"version": "2.6.1",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz",
"integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw=="
},
"node-libs-browser": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/node-libs-browser/-/node-libs-browser-2.2.1.tgz",
@@ -4912,6 +5099,48 @@
"ws": "^7.2.3"
}
},
"puppeteer-extra": {
"version": "3.1.15",
"resolved": "https://registry.npmjs.org/puppeteer-extra/-/puppeteer-extra-3.1.15.tgz",
"integrity": "sha512-TFKcluoNSYCT3xmZjDcqOkBuxZePbwvaL5mrW5Gvp5c9QsJEei5TYixoenMQaB3QZuRW0Aura4yyjVrJDNlWFA==",
"requires": {
"@types/debug": "^4.1.0",
"@types/puppeteer": "*",
"debug": "^4.1.1",
"deepmerge": "^4.2.2"
}
},
"puppeteer-extra-plugin": {
"version": "3.1.7",
"resolved": "https://registry.npmjs.org/puppeteer-extra-plugin/-/puppeteer-extra-plugin-3.1.7.tgz",
"integrity": "sha512-In3o89X06Q4w1wt0RsAvvdRIp7BzWttVHh24srzlfcBvsGq+zCqhPuwbOYbxnwemtNLaiD/hcYGLvNMEqeUo/Q==",
"requires": {
"@types/debug": "^4.1.0",
"debug": "^4.1.1",
"merge-deep": "^3.0.1"
}
},
"puppeteer-extra-plugin-adblocker": {
"version": "2.11.6",
"resolved": "https://registry.npmjs.org/puppeteer-extra-plugin-adblocker/-/puppeteer-extra-plugin-adblocker-2.11.6.tgz",
"integrity": "sha512-tlCmcsdfRr57lzP1tT07hsEHGtv9pYO+1rzV1AtzX3+Tci1M4Y3ANwjrrYCiAx5Sbf5MF6apqaofBvTZBwfPbg==",
"requires": {
"@cliqz/adblocker-puppeteer": "^1.4.0",
"@types/chrome": "0.0.91",
"debug": "^4.1.1",
"node-fetch": "^2.6.0",
"puppeteer-extra-plugin": "^3.1.7"
}
},
"puppeteer-extra-plugin-stealth": {
"version": "2.6.1",
"resolved": "https://registry.npmjs.org/puppeteer-extra-plugin-stealth/-/puppeteer-extra-plugin-stealth-2.6.1.tgz",
"integrity": "sha512-KPoqjvYdnX8BpcjuXZKethad11WY1UfVzckSmOpBCCtdYic1s6esPhenLTvBEfZQ5XAT61yLK8jQtNgkML0QVg==",
"requires": {
"debug": "^4.1.1",
"puppeteer-extra-plugin": "^3.1.7"
}
},
"pushover-notifications": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/pushover-notifications/-/pushover-notifications-1.2.2.tgz",
@@ -5297,6 +5526,32 @@
"safe-buffer": "^5.0.1"
}
},
"shallow-clone": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-0.1.2.tgz",
"integrity": "sha1-WQnodLp3EG1zrEFM/sH/yofZcGA=",
"requires": {
"is-extendable": "^0.1.1",
"kind-of": "^2.0.1",
"lazy-cache": "^0.2.3",
"mixin-object": "^2.0.1"
},
"dependencies": {
"kind-of": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/kind-of/-/kind-of-2.0.1.tgz",
"integrity": "sha1-AY7HpM5+OobLkUG+UZ0kyPqpgbU=",
"requires": {
"is-buffer": "^1.0.2"
}
},
"lazy-cache": {
"version": "0.2.7",
"resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-0.2.7.tgz",
"integrity": "sha1-f+3fLctu23fRHvHRF6tf/fCrG2U="
}
}
},
"shebang-command": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
@@ -5862,6 +6117,19 @@
"setimmediate": "^1.0.4"
}
},
"tldts-core": {
"version": "5.6.55",
"resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-5.6.55.tgz",
"integrity": "sha512-xAUqjkPfwfnCoRxds3xrMi6j1nuR1F4iuB9dqE30sgi0P+7c4NfnbwQTk80Vu1CL15ZyRZeZ+yAwWo2rWxkC9w=="
},
"tldts-experimental": {
"version": "5.6.55",
"resolved": "https://registry.npmjs.org/tldts-experimental/-/tldts-experimental-5.6.55.tgz",
"integrity": "sha512-GKfOgP1XxHgVcGp9fIA/dPzr2cSUpS5hCpJBvkpTZ8hwOj5UX7mSUSO8tag7udWdmIDh6FdJhIPCzmDaM9n7WQ==",
"requires": {
"tldts-core": "^5.6.55"
}
},
"to-absolute-glob": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/to-absolute-glob/-/to-absolute-glob-2.0.2.tgz",