feat: enhanced lookup behaviour (#270)

Co-authored-by: Jef LeCompte <jeffreylec@gmail.com>

.env-example

@@ -20,6 +20,8 @@ PUSHBULLET=""
 PUSHOVER_TOKEN=""
 PUSHOVER_USER=""
 PUSHOVER_PRIORITY=""
+PAGE_BACKOFF_MIN=""
+PAGE_BACKOFF_MAX=""
 PAGE_SLEEP_MIN=""
 PAGE_SLEEP_MAX=""
 SHOW_ONLY_BRANDS=""

README.md

@@ -56,6 +56,10 @@ At any point you want the program to stop, use <kbd>Ctrl</kbd> + <kbd>C</kbd>.
 
 > :point_right: Please visit the [wiki](https://github.com/jef/nvidia-snatcher/wiki) if you need more help with installation.
 
+### Developer notes
+
+The command `npm run dev` can be used instead of `npm run start` to automatically restart the project when filesystem changes are detected in the `src/` folder or `.env` file.
+
 ### Customization
 
 To customize `nvidia-snatcher`, make a copy of `.env-example` as `.env` and make any changes to your liking. _All environment variables are **optional**._
@@ -83,6 +87,8 @@ Here is a list of variables that you can use to customize your newly copied `.en
 | `PUSHOVER_TOKEN` | Pushover access token | Generate at https://pushover.net/apps/build | |
 | `PUSHOVER_USER` | Pushover username | |
 | `PUSHOVER_PRIORITY` | Pushover message priority |
+| `PAGE_BACKOFF_MIN` | Minimum backoff time between retrying requests for the same store when a forbidden response is received | Default: `10000` |
+| `PAGE_BACKOFF_MAX` | Maximum backoff time between retrying requests for the same store when a forbidden response is received | Default: `3600000` |
 | `PAGE_SLEEP_MIN` | Minimum sleep time between queries of the same store | In milliseconds, default: `5000` |
 | `PAGE_SLEEP_MAX` | Maximum sleep time between queries of the same store | In milliseconds, default: `10000` |
 | `SCREENSHOT` | Capture screenshot of page if a card is found | Default: `true` |

nodemon.json

@@ -0,0 +1,8 @@
+{
+  "exec": "ts-node --files ./src/index",
+  "ext": "ts",
+  "watch": [
+    "src/",
+    ".env"
+  ]
+}

package-lock.json

@@ -313,7 +313,6 @@
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/@slack/logger/-/logger-2.0.0.tgz",
       "integrity": "sha512-OkIJpiU2fz6HOJujhlhfIGrc8hB4ibqtf7nnbJQDerG0BqwZCfmgtK5sWzZ0TkXVRBKD5MpLrTmCYyMxoMCgPw==",
-      "dev": true,
       "requires": {
         "@types/node": ">=8.9.0"
       }
@@ -321,14 +320,12 @@
     "@slack/types": {
       "version": "1.9.0",
       "resolved": "https://registry.npmjs.org/@slack/types/-/types-1.9.0.tgz",
-      "integrity": "sha512-RmwgMWqOtzd2JPXdiaD/tyrDD0vtjjRDFdxN1I3tAxwBbg4aryzDUVqFc8na16A+3Xik/UN8X1hvVTw8J4EB9w==",
+      "integrity": "sha512-RmwgMWqOtzd2JPXdiaD/tyrDD0vtjjRDFdxN1I3tAxwBbg4aryzDUVqFc8na16A+3Xik/UN8X1hvVTw8J4EB9w=="
-      "dev": true
     },
     "@slack/web-api": {
       "version": "5.12.0",
       "resolved": "https://registry.npmjs.org/@slack/web-api/-/web-api-5.12.0.tgz",
       "integrity": "sha512-ygSnNHVid7PltGo7W36f2SNVHyliemkzxn9uSwgnWNF7CHmWBKWAylU/eoDml9l5K7akMOxbousiurOw4XqOFg==",
-      "dev": true,
       "requires": {
         "@slack/logger": ">=1.0.0 <3.0.0",
         "@slack/types": "^1.7.0",
@@ -346,8 +343,7 @@
         "is-stream": {
           "version": "1.1.0",
           "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz",
-          "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=",
+          "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ="
-          "dev": true
         }
       }
     },
@@ -372,6 +368,15 @@
       "integrity": "sha512-6ckxMjBBD8URvjB6J3NcnuAn5Pkl7t3TizAg+xdlzzQGSPSmBcXf8KoIH0ua/i+tio+ZRUHEXp0HEmvaR4kt0w==",
       "dev": true
     },
+    "@types/cheerio": {
+      "version": "0.22.22",
+      "resolved": "https://registry.npmjs.org/@types/cheerio/-/cheerio-0.22.22.tgz",
+      "integrity": "sha512-05DYX4zU96IBfZFY+t3Mh88nlwSMtmmzSYaQkKN48T495VV1dkHSah6qYyDTN5ngaS0i0VonH37m+RuzSM0YiA==",
+      "dev": true,
+      "requires": {
+        "@types/node": "*"
+      }
+    },
     "@types/chrome": {
       "version": "0.0.91",
       "resolved": "https://registry.npmjs.org/@types/chrome/-/chrome-0.0.91.tgz",
@@ -433,7 +438,6 @@
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/@types/is-stream/-/is-stream-1.1.0.tgz",
       "integrity": "sha512-jkZatu4QVbR60mpIzjINmtS1ZF4a/FqdTUTBeQDVOQ2PYyidtwFKr0B5G6ERukKwliq+7mIXvxyppwzG5EgRYg==",
-      "dev": true,
       "requires": {
         "@types/node": "*"
       }
@@ -499,8 +503,7 @@
     "@types/p-queue": {
       "version": "2.3.2",
       "resolved": "https://registry.npmjs.org/@types/p-queue/-/p-queue-2.3.2.tgz",
-      "integrity": "sha512-eKAv5Ql6k78dh3ULCsSBxX6bFNuGjTmof5Q/T6PiECDq0Yf8IIn46jCyp3RJvCi8owaEmm3DZH1PEImjBMd/vQ==",
+      "integrity": "sha512-eKAv5Ql6k78dh3ULCsSBxX6bFNuGjTmof5Q/T6PiECDq0Yf8IIn46jCyp3RJvCi8owaEmm3DZH1PEImjBMd/vQ=="
-      "dev": true
     },
     "@types/parse-json": {
       "version": "4.0.0",
@@ -531,8 +534,7 @@
     "@types/retry": {
       "version": "0.12.0",
       "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz",
-      "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==",
+      "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA=="
-      "dev": true
     },
     "@types/tough-cookie": {
       "version": "4.0.0",
@@ -674,6 +676,12 @@
         "eslint-visitor-keys": "^1.1.0"
       }
     },
+    "abbrev": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
+      "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==",
+      "dev": true
+    },
     "acorn": {
       "version": "7.4.0",
       "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.0.tgz",
@@ -749,6 +757,22 @@
         "color-convert": "^1.9.0"
       }
     },
+    "anymatch": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.1.tgz",
+      "integrity": "sha512-mM8522psRCqzV+6LhomX5wgp25YVibjh8Wj23I5RPkPppSVSjyKD2A2mBJmWGa+KN7f2D6LNh9jkBCeyLktzjg==",
+      "dev": true,
+      "requires": {
+        "normalize-path": "^3.0.0",
+        "picomatch": "^2.0.4"
+      }
+    },
+    "arg": {
+      "version": "4.1.3",
+      "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz",
+      "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==",
+      "dev": true
+    },
     "argparse": {
       "version": "1.0.10",
       "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
@@ -1042,6 +1066,12 @@
         "tweetnacl": "^0.14.3"
       }
     },
+    "binary-extensions": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.1.0.tgz",
+      "integrity": "sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ==",
+      "dev": true
+    },
     "bl": {
       "version": "4.0.3",
       "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.3.tgz",
@@ -1058,6 +1088,11 @@
       "integrity": "sha512-GkTiFpjFtUzU9CbMeJ5iazkCzGL3jrhzerzZIuqLABjbwRaFt33I9tUdSNryIptM+RxDet6OKm2WnLXzW51KsQ==",
       "dev": true
     },
+    "boolbase": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
+      "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24="
+    },
     "boxen": {
       "version": "4.2.0",
       "resolved": "https://registry.npmjs.org/boxen/-/boxen-4.2.0.tgz",
@@ -1294,6 +1329,12 @@
       "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
       "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI="
     },
+    "buffer-from": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz",
+      "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==",
+      "dev": true
+    },
     "buffer-xor": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz",
@@ -1422,6 +1463,70 @@
         }
       }
     },
+    "cheerio": {
+      "version": "1.0.0-rc.3",
+      "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.3.tgz",
+      "integrity": "sha512-0td5ijfUPuubwLUu0OBoe98gZj8C/AA+RW3v67GPlGOrvxWjZmBXiBCRU+I8VEiNyJzjth40POfHiz2RB3gImA==",
+      "requires": {
+        "css-select": "~1.2.0",
+        "dom-serializer": "~0.1.1",
+        "entities": "~1.1.1",
+        "htmlparser2": "^3.9.1",
+        "lodash": "^4.15.0",
+        "parse5": "^3.0.1"
+      }
+    },
+    "chokidar": {
+      "version": "3.4.2",
+      "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.4.2.tgz",
+      "integrity": "sha512-IZHaDeBeI+sZJRX7lGcXsdzgvZqKv6sECqsbErJA4mHWfpRrD8B97kSFN4cQz6nGBGiuFia1MKR4d6c1o8Cv7A==",
+      "dev": true,
+      "requires": {
+        "anymatch": "~3.1.1",
+        "braces": "~3.0.2",
+        "fsevents": "~2.1.2",
+        "glob-parent": "~5.1.0",
+        "is-binary-path": "~2.1.0",
+        "is-glob": "~4.0.1",
+        "normalize-path": "~3.0.0",
+        "readdirp": "~3.4.0"
+      },
+      "dependencies": {
+        "braces": {
+          "version": "3.0.2",
+          "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz",
+          "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==",
+          "dev": true,
+          "requires": {
+            "fill-range": "^7.0.1"
+          }
+        },
+        "fill-range": {
+          "version": "7.0.1",
+          "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
+          "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==",
+          "dev": true,
+          "requires": {
+            "to-regex-range": "^5.0.1"
+          }
+        },
+        "is-number": {
+          "version": "7.0.0",
+          "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
+          "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
+          "dev": true
+        },
+        "to-regex-range": {
+          "version": "5.0.1",
+          "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
+          "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
+          "dev": true,
+          "requires": {
+            "is-number": "^7.0.0"
+          }
+        }
+      }
+    },
     "chownr": {
       "version": "1.1.4",
       "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
@@ -1756,6 +1861,22 @@
       "integrity": "sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA==",
       "dev": true
     },
+    "css-select": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz",
+      "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=",
+      "requires": {
+        "boolbase": "~1.0.0",
+        "css-what": "2.1",
+        "domutils": "1.5.1",
+        "nth-check": "~1.0.1"
+      }
+    },
+    "css-what": {
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.3.tgz",
+      "integrity": "sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg=="
+    },
     "dashdash": {
       "version": "1.14.1",
       "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
@@ -1913,6 +2034,12 @@
       "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.799653.tgz",
       "integrity": "sha512-t1CcaZbvm8pOlikqrsIM9GOa7Ipp07+4h/q9u0JXBWjPCjHdBl9KkddX87Vv9vBHoBGtwV79sYQNGnQM6iS5gg=="
     },
+    "diff": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
+      "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==",
+      "dev": true
+    },
     "diffie-hellman": {
       "version": "5.0.3",
       "resolved": "https://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz",
@@ -1962,7 +2089,6 @@
       "version": "1.1.8",
       "resolved": "https://registry.npmjs.org/discord-webhook-node/-/discord-webhook-node-1.1.8.tgz",
       "integrity": "sha512-3u0rrwywwYGc6HrgYirN/9gkBYqmdpvReyQjapoXARAHi0P0fIyf3W5tS5i3U3cc7e44E+e7dIHYUeec7yWaug==",
-      "dev": true,
       "requires": {
         "form-data": "^3.0.0",
         "node-fetch": "^2.6.0"
@@ -1972,7 +2098,6 @@
           "version": "3.0.0",
           "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.0.tgz",
           "integrity": "sha512-CKMFDglpbMi6PyN+brwB9Q/GOw0eAnsrEZDgcsH5Krhz5Od/haKHAX0NmQfha2zPPz0JpWzA7GJHGSnvCRLWsg==",
-          "dev": true,
           "requires": {
             "asynckit": "^0.4.0",
             "combined-stream": "^1.0.8",
@@ -1990,12 +2115,43 @@
         "esutils": "^2.0.2"
       }
     },
+    "dom-serializer": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.1.tgz",
+      "integrity": "sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==",
+      "requires": {
+        "domelementtype": "^1.3.0",
+        "entities": "^1.1.1"
+      }
+    },
     "domain-browser": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/domain-browser/-/domain-browser-1.2.0.tgz",
       "integrity": "sha512-jnjyiM6eRyZl2H+W8Q/zLMA481hzi0eszAaBUzIVnmYVDBbnLxVNnfu1HgEBvCbL+71FrxMl3E6lpKH7Ge3OXA==",
       "dev": true
     },
+    "domelementtype": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz",
+      "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w=="
+    },
+    "domhandler": {
+      "version": "2.4.2",
+      "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz",
+      "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==",
+      "requires": {
+        "domelementtype": "1"
+      }
+    },
+    "domutils": {
+      "version": "1.5.1",
+      "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz",
+      "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=",
+      "requires": {
+        "dom-serializer": "0",
+        "domelementtype": "1"
+      }
+    },
     "dot-case": {
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/dot-case/-/dot-case-3.0.3.tgz",
@@ -2105,6 +2261,11 @@
         "ansi-colors": "^4.1.1"
       }
     },
+    "entities": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz",
+      "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w=="
+    },
     "env-editor": {
       "version": "0.4.1",
       "resolved": "https://registry.npmjs.org/env-editor/-/env-editor-0.4.1.tgz",
@@ -2853,8 +3014,7 @@
     "eventemitter3": {
       "version": "3.1.2",
       "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-3.1.2.tgz",
-      "integrity": "sha512-tvtQIeLVHjDkJYnzf2dgVMxfuSGJeM/7UCG17TT4EumTfNtF+0nebF/4zWOIkCreAbtNqhGEboB6BWrwqNaw4Q==",
+      "integrity": "sha512-tvtQIeLVHjDkJYnzf2dgVMxfuSGJeM/7UCG17TT4EumTfNtF+0nebF/4zWOIkCreAbtNqhGEboB6BWrwqNaw4Q=="
-      "dev": true
     },
     "events": {
       "version": "3.2.0",
@@ -3182,8 +3342,7 @@
     "find-exec": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/find-exec/-/find-exec-1.0.1.tgz",
-      "integrity": "sha512-4o6QkGkpg3xK5s/47rdK9LDZRsE4JR1mrXnaAOXBngG6UKeIDJXfwtNCAkljgyy6VRh75D3FFaB0tii9vDEtIA==",
+      "integrity": "sha512-4o6QkGkpg3xK5s/47rdK9LDZRsE4JR1mrXnaAOXBngG6UKeIDJXfwtNCAkljgyy6VRh75D3FFaB0tii9vDEtIA=="
-      "dev": true
     },
     "find-root": {
       "version": "1.1.0",
@@ -3272,7 +3431,6 @@
       "version": "2.5.1",
       "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz",
       "integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==",
-      "dev": true,
       "requires": {
         "asynckit": "^0.4.0",
         "combined-stream": "^1.0.6",
@@ -3310,6 +3468,13 @@
       "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
       "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
     },
+    "fsevents": {
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.1.3.tgz",
+      "integrity": "sha512-Auw9a4AxqWpa9GUfj370BMPzzyncfBABW8Mab7BGWBYDj4Isgq+cDKtx0i6u9jcX9pQDnswsaaOTgTmA5pEjuQ==",
+      "dev": true,
+      "optional": true
+    },
     "function-bind": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
@@ -3597,6 +3762,19 @@
       "integrity": "sha512-f/wzC2QaWBs7t9IYqB4T3sR1xviIViXJRJTWBlx2Gf3g0Xi5vI7Yy4koXQ1c9OYDGHN9sBy1DQ2AB8fqZBWhUg==",
       "dev": true
     },
+    "htmlparser2": {
+      "version": "3.10.1",
+      "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz",
+      "integrity": "sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==",
+      "requires": {
+        "domelementtype": "^1.3.1",
+        "domhandler": "^2.3.0",
+        "domutils": "^1.5.1",
+        "entities": "^1.1.1",
+        "inherits": "^2.0.1",
+        "readable-stream": "^3.1.1"
+      }
+    },
     "http-cache-semantics": {
       "version": "4.1.0",
       "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz",
@@ -3718,6 +3896,12 @@
       "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==",
       "dev": true
     },
+    "ignore-by-default": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz",
+      "integrity": "sha1-SMptcvbGo68Aqa1K5odr44ieKwk=",
+      "dev": true
+    },
     "import-fresh": {
       "version": "3.2.1",
       "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.1.tgz",
@@ -3827,6 +4011,15 @@
       "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz",
       "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ=="
     },
+    "is-binary-path": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz",
+      "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==",
+      "dev": true,
+      "requires": {
+        "binary-extensions": "^2.0.0"
+      }
+    },
     "is-buffer": {
       "version": "1.1.6",
       "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
@@ -4457,6 +4650,12 @@
         }
       }
     },
+    "make-error": {
+      "version": "1.3.6",
+      "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz",
+      "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==",
+      "dev": true
+    },
     "map-cache": {
       "version": "0.2.2",
       "resolved": "https://registry.npmjs.org/map-cache/-/map-cache-0.2.2.tgz",
@@ -4984,6 +5183,71 @@
       "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-6.4.11.tgz",
       "integrity": "sha512-BVZBDi+aJV4O38rxsUh164Dk1NCqgh6Cm0rQSb9SK/DHGll/DrCMnycVDD7msJgZCnmVa8ASo8EZzR7jsgTukQ=="
     },
+    "nodemon": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-2.0.4.tgz",
+      "integrity": "sha512-Ltced+hIfTmaS28Zjv1BM552oQ3dbwPqI4+zI0SLgq+wpJhSyqgYude/aZa/3i31VCQWMfXJVxvu86abcam3uQ==",
+      "dev": true,
+      "requires": {
+        "chokidar": "^3.2.2",
+        "debug": "^3.2.6",
+        "ignore-by-default": "^1.0.1",
+        "minimatch": "^3.0.4",
+        "pstree.remy": "^1.1.7",
+        "semver": "^5.7.1",
+        "supports-color": "^5.5.0",
+        "touch": "^3.1.0",
+        "undefsafe": "^2.0.2",
+        "update-notifier": "^4.0.0"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "3.2.6",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz",
+          "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==",
+          "dev": true,
+          "requires": {
+            "ms": "^2.1.1"
+          }
+        },
+        "has-flag": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
+          "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=",
+          "dev": true
+        },
+        "ms": {
+          "version": "2.1.2",
+          "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+          "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==",
+          "dev": true
+        },
+        "semver": {
+          "version": "5.7.1",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
+          "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==",
+          "dev": true
+        },
+        "supports-color": {
+          "version": "5.5.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
+          "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
+          "dev": true,
+          "requires": {
+            "has-flag": "^3.0.0"
+          }
+        }
+      }
+    },
+    "nopt": {
+      "version": "1.0.10",
+      "resolved": "https://registry.npmjs.org/nopt/-/nopt-1.0.10.tgz",
+      "integrity": "sha1-bd0hvSoxQXuScn3Vhfim83YI6+4=",
+      "dev": true,
+      "requires": {
+        "abbrev": "1"
+      }
+    },
     "normalize-package-data": {
       "version": "2.5.0",
       "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz",
@@ -5004,12 +5268,26 @@
         }
       }
     },
+    "normalize-path": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz",
+      "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==",
+      "dev": true
+    },
     "normalize-url": {
       "version": "4.5.0",
       "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.0.tgz",
       "integrity": "sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ==",
       "dev": true
     },
+    "nth-check": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz",
+      "integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==",
+      "requires": {
+        "boolbase": "~1.0.0"
+      }
+    },
     "oauth-sign": {
       "version": "0.9.0",
       "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz",
@@ -5238,8 +5516,7 @@
     "p-queue": {
       "version": "2.4.2",
       "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-2.4.2.tgz",
-      "integrity": "sha512-n8/y+yDJwBjoLQe1GSJbbaYQLTI7QHNZI2+rpmCDbe++WLf9HC3gf6iqj5yfPAV71W4UF3ql5W1+UBPXoXTxng==",
+      "integrity": "sha512-n8/y+yDJwBjoLQe1GSJbbaYQLTI7QHNZI2+rpmCDbe++WLf9HC3gf6iqj5yfPAV71W4UF3ql5W1+UBPXoXTxng=="
-      "dev": true
     },
     "p-reduce": {
       "version": "2.1.0",
@@ -5251,7 +5528,6 @@
       "version": "4.2.0",
       "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.2.0.tgz",
       "integrity": "sha512-jPH38/MRh263KKcq0wBNOGFJbm+U6784RilTmHjB/HM9kH9V8WlCpVUcdOmip9cjXOh6MxZ5yk1z2SjDUJfWmA==",
-      "dev": true,
       "requires": {
         "@types/retry": "^0.12.0",
         "retry": "^0.12.0"
@@ -5322,6 +5598,14 @@
         "lines-and-columns": "^1.1.6"
       }
     },
+    "parse5": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz",
+      "integrity": "sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==",
+      "requires": {
+        "@types/node": "*"
+      }
+    },
     "pascal-case": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/pascal-case/-/pascal-case-3.1.1.tgz",
@@ -5424,7 +5708,6 @@
       "version": "1.1.3",
       "resolved": "https://registry.npmjs.org/play-sound/-/play-sound-1.1.3.tgz",
       "integrity": "sha512-lqEzgtNAdfg2VUXItOtu5bTyWcqeFmnJmgvc8iHEeEOBEJdurqiGYfmCOzIpSBcwrs7XeSpvHv+Rw9dzRPc4aw==",
-      "dev": true,
       "requires": {
         "find-exec": "1.0.1"
       }
@@ -5518,6 +5801,12 @@
       "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz",
       "integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ=="
     },
+    "pstree.remy": {
+      "version": "1.1.8",
+      "resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.8.tgz",
+      "integrity": "sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==",
+      "dev": true
+    },
     "public-encrypt": {
       "version": "4.0.3",
       "resolved": "https://registry.npmjs.org/public-encrypt/-/public-encrypt-4.0.3.tgz",
@@ -5881,6 +6170,15 @@
         "util-deprecate": "^1.0.1"
       }
     },
+    "readdirp": {
+      "version": "3.4.0",
+      "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.4.0.tgz",
+      "integrity": "sha512-0xe001vZBnJEK+uKcj8qOhyAKPzIT+gStxWr3LCB0DwcXR5NZJ3IaC+yGnHCYzB/S7ov3m3EEbZI2zeNvX+hGQ==",
+      "dev": true,
+      "requires": {
+        "picomatch": "^2.2.1"
+      }
+    },
     "redent": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/redent/-/redent-3.0.0.tgz",
@@ -6070,8 +6368,7 @@
     "retry": {
       "version": "0.12.0",
       "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz",
-      "integrity": "sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs=",
+      "integrity": "sha1-G0KmJmoh8HQh0bC1S33BZ7AcATs="
-      "dev": true
     },
     "rimraf": {
       "version": "3.0.2",
@@ -6404,6 +6701,24 @@
         "urix": "^0.1.0"
       }
     },
+    "source-map-support": {
+      "version": "0.5.19",
+      "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz",
+      "integrity": "sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==",
+      "dev": true,
+      "requires": {
+        "buffer-from": "^1.0.0",
+        "source-map": "^0.6.0"
+      },
+      "dependencies": {
+        "source-map": {
+          "version": "0.6.1",
+          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
+          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
+          "dev": true
+        }
+      }
+    },
     "source-map-url": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/source-map-url/-/source-map-url-0.4.0.tgz",
@@ -6872,6 +7187,15 @@
         "repeat-string": "^1.6.1"
       }
     },
+    "touch": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/touch/-/touch-3.1.0.tgz",
+      "integrity": "sha512-WBx8Uy5TLtOSRtIq+M03/sKDrXCLHxwDcquSP2c43Le03/9serjQBIztjRz6FkJez9D/hleyAXTBGLwwZUw9lA==",
+      "dev": true,
+      "requires": {
+        "nopt": "~1.0.10"
+      }
+    },
     "tough-cookie": {
       "version": "2.5.0",
       "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz",
@@ -6900,6 +7224,19 @@
         "tslib": "^1.9.3"
       }
     },
+    "ts-node": {
+      "version": "9.0.0",
+      "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-9.0.0.tgz",
+      "integrity": "sha512-/TqB4SnererCDR/vb4S/QvSZvzQMJN8daAslg7MeaiHvD8rDZsSfXmNeNumyZZzMned72Xoq/isQljYSt8Ynfg==",
+      "dev": true,
+      "requires": {
+        "arg": "^4.1.0",
+        "diff": "^4.0.1",
+        "make-error": "^1.1.1",
+        "source-map-support": "^0.5.17",
+        "yn": "3.1.1"
+      }
+    },
     "tsconfig-paths": {
       "version": "3.9.0",
       "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.9.0.tgz",
@@ -7016,6 +7353,26 @@
       "integrity": "sha1-5z3T17DXxe2G+6xrCufYxqadUPo=",
       "dev": true
     },
+    "undefsafe": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.3.tgz",
+      "integrity": "sha512-nrXZwwXrD/T/JXeygJqdCO6NZZ1L66HrxM/Z7mIq2oPanoN0F1nLx3lwJMu6AwJY69hdixaFQOuoYsMjE5/C2A==",
+      "dev": true,
+      "requires": {
+        "debug": "^2.2.0"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "2.6.9",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
+          "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
+          "dev": true,
+          "requires": {
+            "ms": "2.0.0"
+          }
+        }
+      }
+    },
     "union-value": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz",
@@ -7518,6 +7875,12 @@
         "buffer-crc32": "~0.2.3",
         "fd-slicer": "~1.1.0"
       }
+    },
+    "yn": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz",
+      "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==",
+      "dev": true
     }
   }
 }

package.json

@@ -5,6 +5,7 @@
   "main": "src/index.ts",
   "scripts": {
     "build": "rimraf ./build && tsc",
+	"dev": "nodemon --config nodemon.json",
     "lint": "xo",
     "lint:fix": "xo --fix",
     "start": "npm run build && node build/index.js",
@@ -22,12 +23,16 @@
   },
   "homepage": "https://github.com/jef/nvidia-snatcher#readme",
   "dependencies": {
+    "@slack/web-api": "^5.12.0",
     "chalk": "^4.1.0",
+    "cheerio": "^1.0.0-rc.3",
+    "discord-webhook-node": "^1.1.8",
     "dotenv": "^8.2.0",
     "messaging-api-telegram": "^1.0.1",
     "node-notifier": "^8.0.0",
     "nodemailer": "^6.4.11",
     "open": "^7.2.1",
+    "play-sound": "^1.1.3",
     "puppeteer": "^5.3.1",
     "puppeteer-extra": "^3.1.15",
     "puppeteer-extra-plugin-adblocker": "^2.11.6",
@@ -38,17 +43,17 @@
     "winston": "^3.3.3"
   },
   "devDependencies": {
-    "@slack/web-api": "^5.12.0",
     "@types/async": "^3.2.3",
+    "@types/cheerio": "^0.22.22",
     "@types/node": "^14.11.2",
     "@types/node-notifier": "^8.0.0",
     "@types/nodemailer": "^6.4.0",
     "@types/puppeteer": "^3.0.2",
     "@types/twitter": "^1.7.0",
-    "discord-webhook-node": "^1.1.8",
     "husky": "^4.3.0",
-    "play-sound": "^1.1.3",
+    "nodemon": "^2.0.4",
     "rimraf": "^3.0.2",
+    "ts-node": "^9.0.0",
     "typescript": "^4.0.2",
     "xo": "^0.33.1"
   },

src/adblocker.ts

@@ -7,5 +7,7 @@ export const adBlocker = new PuppeteerExtraPluginAdblocker({
 
 export async function disableBlockerInPage(page: Page) {
 	const blockerObject = await adBlocker.getBlocker();
-	await blockerObject.disableBlockingInPage(page);
+	if (blockerObject.isBlockingEnabled(page)) {
+		await blockerObject.disableBlockingInPage(page);
+	}
 }

src/config.ts

@@ -49,7 +49,9 @@ function envOrNumber(environment: string | undefined, number?: number): number {
 const browser = {
 	isHeadless: envOrBoolean(process.env.HEADLESS),
 	isTrusted: envOrBoolean(process.env.BROWSER_TRUSTED, false),
+	maxBackoff: envOrNumber(process.env.PAGE_BACKOFF_MAX, 3600000),
 	maxSleep: envOrNumber(process.env.PAGE_SLEEP_MAX, 10000),
+	minBackoff: envOrNumber(process.env.PAGE_BACKOFF_MIN, 10000),
 	minSleep: envOrNumber(process.env.PAGE_SLEEP_MIN, 5000),
 	open: envOrBoolean(process.env.OPEN_BROWSER)
 };

src/index.ts

@@ -2,6 +2,7 @@ import {Config} from './config';
 import {Logger} from './logger';
 import {Stores} from './store/model';
 import {adBlocker} from './adblocker';
+import {fetchLinks} from './store/fetch-links';
 import {getSleepTime} from './util';
 import puppeteer from 'puppeteer-extra';
 import stealthPlugin from 'puppeteer-extra-plugin-stealth';
@@ -37,14 +38,21 @@ async function main() {
 		headless: Config.browser.isHeadless
 	});
 
+	const promises = [];
 	for (const store of Stores) {
 		Logger.debug(store.links);
 		if (store.setupAction !== undefined) {
 			store.setupAction(browser);
 		}
 
+		if (store.linksBuilder) {
+			promises.push(fetchLinks(store, browser));
+		}
+
 		setTimeout(tryLookupAndLoop, getSleepTime(), browser, store);
 	}
+
+	await Promise.all(promises);
 }
 
 /**

src/logger.ts

@@ -26,6 +26,27 @@ export const Logger = winston.createLogger({
 });
 
 export const Print = {
+	backoff(link: Link, store: Store, delay: number, color?: boolean): string {
+		if (color) {
+			return '✖ ' + buildProductString(link, store, true) + ' :: ' + chalk.yellow(`REQUEST FORBIDDEN - BACKOFF DELAY ${delay}`);
+		}
+
+		return `✖ ${buildProductString(link, store)} :: REQUEST FORBIDDEN - BACKOFF DELAY ${delay}`;
+	},
+	badStatusCode(link: Link, store: Store, statusCode: number, color?: boolean): string {
+		if (color) {
+			return '✖ ' + buildProductString(link, store, true) + ' :: ' + chalk.yellow(`STATUS CODE ERROR ${statusCode}`);
+		}
+
+		return `✖ ${buildProductString(link, store)} :: STATUS CODE ERROR ${statusCode}`;
+	},
+	bannedSeller(link: Link, store: Store, color?: boolean): string {
+		if (color) {
+			return '✖ ' + buildProductString(link, store, true) + ' :: ' + chalk.yellow('BANNED SELLER');
+		}
+
+		return `✖ ${buildProductString(link, store)} :: BANNED SELLER`;
+	},
 	captcha(link: Link, store: Store, color?: boolean): string {
 		if (color) {
 			return '✖ ' + buildProductString(link, store, true) + ' :: ' + chalk.yellow('CAPTCHA');
@@ -47,6 +68,20 @@ export const Print = {
 
 		return `ℹ ${buildProductString(link, store)} :: IN STOCK, WAITING`;
 	},
+	message(message: string, topic: string, store: Store, color?: boolean): string {
+		if (color) {
+			return '✖ ' + buildSetupString(topic, store, true) + ' :: ' + chalk.yellow(message);
+		}
+
+		return `✖ ${buildSetupString(topic, store)} :: ${message}`;
+	},
+	noResponse(link: Link, store: Store, color?: boolean): string {
+		if (color) {
+			return '✖ ' + buildProductString(link, store, true) + ' :: ' + chalk.yellow('NO RESPONSE');
+		}
+
+		return `✖ ${buildProductString(link, store)} :: NO RESPONSE`;
+	},
 	outOfStock(link: Link, store: Store, color?: boolean): string {
 		if (color) {
 			return '✖ ' + buildProductString(link, store, true) + ' :: ' + chalk.red('OUT OF STOCK');
@@ -63,6 +98,14 @@ export const Print = {
 	}
 };
 
+function buildSetupString(topic: string, store: Store, color?: boolean): string {
+	if (color) {
+		return chalk.cyan(`[${store.name}]`) + chalk.grey(` [setup (${topic})]`);
+	}
+
+	return `[${store.name}] [setup (${topic})]`;
+}
+
 function buildProductString(link: Link, store: Store, color?: boolean): string {
 	if (color) {
 		return chalk.cyan(`[${store.name}]`) + chalk.grey(` [${link.brand} (${link.series})] ${link.model}`);

src/store/fetch-links.ts

@@ -0,0 +1,58 @@
+import {Link, Series, Store} from './model';
+import {Logger, Print} from '../logger';
+import {Browser} from 'puppeteer';
+import cheerio from 'cheerio';
+import {filterSeries} from './filter';
+import {usingResponse} from '../util';
+
+function addNewLinks(store: Store, links: Link[], series: Series) {
+	if (links.length === 0) {
+		Logger.error(Print.message('NO STORE LINKS FOUND', series, store, true));
+
+		return;
+	}
+
+	const existingUrls = new Set(store.links.map(link => link.url));
+	const newLinks = links.filter(link => !existingUrls.has(link.url));
+
+	if (newLinks.length === 0) {
+		return;
+	}
+
+	Logger.info(Print.message(`FOUND ${newLinks.length} STORE LINKS`, series, store, true));
+	Logger.debug(JSON.stringify(newLinks, null, 2));
+
+	store.links = store.links.concat(newLinks);
+}
+
+export async function fetchLinks(store: Store, browser: Browser) {
+	if (!store.linksBuilder) {
+		return;
+	}
+
+	const promises = [];
+
+	for (const {series, url} of store.linksBuilder.urls) {
+		if (!filterSeries(series)) {
+			continue;
+		}
+
+		Logger.info(Print.message('DETECTING STORE LINKS', series, store, true));
+
+		promises.push(usingResponse(browser, url, async response => {
+			const text = await response?.text();
+
+			if (!text) {
+				Logger.error(Print.message('NO RESPONSE', series, store, true));
+				return;
+			}
+
+			const docElement = cheerio.load(text).root();
+			const links = store.linksBuilder!.builder(docElement, series);
+
+			addNewLinks(store, links, series);
+		}));
+	}
+
+	await Promise.all(promises);
+}

src/store/filter.ts

@@ -40,7 +40,7 @@ function filterModel(model: Link['model']): boolean {
  *
  * @param series The series of the GPU
  */
-function filterSeries(series: Link['series']): boolean {
+export function filterSeries(series: Link['series']): boolean {
 	if (Config.store.showOnlySeries.length === 0) {
 		return true;
 	}

src/store/includes-labels.ts

@@ -1,3 +1,85 @@
+import {Element, LabelQuery} from './model';
+import {Logger} from '../logger';
+import {Page} from 'puppeteer';
+
+export type Selector = {
+	requireVisible: boolean;
+	selector: string;
+	type: 'innerHTML' | 'outerHTML' | 'textContent';
+};
+
+function isElementArray(query: LabelQuery): query is Element[] {
+	return Array.isArray(query) && query.length > 0 && typeof query[0] === 'object';
+}
+
+function getQueryAsElementArray(query: LabelQuery, defaultContainer: string): Array<Required<Element>> {
+	if (isElementArray(query)) {
+		return query.map(x => ({
+			container: x.container ?? defaultContainer,
+			text: x.text
+		}));
+	}
+
+	if (Array.isArray(query)) {
+		return [{
+			container: defaultContainer,
+			text: query
+		}];
+	}
+
+	return [{
+		container: query.container ?? defaultContainer,
+		text: query.text
+	}];
+}
+
+export async function pageIncludesLabels(page: Page, query: LabelQuery, options: Selector) {
+	const elementQueries = getQueryAsElementArray(query, options.selector);
+
+	const resolved = await Promise.all(elementQueries.map(async query => {
+		const selector = {...options, selector: query.container};
+		const contents = await extractPageContents(page, selector) ?? '';
+
+		if (!contents) {
+			return false;
+		}
+
+		Logger.debug(contents);
+
+		return includesLabels(contents, query.text);
+	}));
+
+	return resolved.includes(true);
+}
+
+export async function extractPageContents(page: Page, selector: Selector): Promise<string | null> {
+	const content = await page.evaluate((options: Selector) => {
+		// eslint-disable-next-line no-undef
+		const element: globalThis.HTMLElement | null = document.querySelector(options.selector);
+
+		if (!element) {
+			return null;
+		}
+
+		if (options.requireVisible && !(element.offsetWidth > 0 && element.offsetHeight > 0)) {
+			return null;
+		}
+
+		switch (options.type) {
+			case 'innerHTML':
+				return element.innerHTML;
+			case 'outerHTML':
+				return element.outerHTML;
+			case 'textContent':
+				return element.textContent;
+			default:
+				return 'Error: selector.type is unknown';
+		}
+	}, selector);
+
+	return content;
+}
+
 /**
  * Checks if DOM has any related text.
  *

src/store/lookup.ts

@@ -1,15 +1,23 @@
 import {Browser, Page, Response} from 'puppeteer';
 import {Link, Store} from './model';
 import {Logger, Print} from '../logger';
+import {Selector, pageIncludesLabels} from './includes-labels';
 import {closePage, delay, getSleepTime} from '../util';
 import {Config} from '../config';
+import {disableBlockerInPage} from '../adblocker';
 import {filterStoreLink} from './filter';
-import {includesLabels} from './includes-labels';
 import open from 'open';
 import {sendNotification} from '../notification';
 
+type Backoff = {
+	count: number;
+	time: number;
+};
+
 const inStock: Record<string, boolean> = {};
 
+const storeBackoff: Record<string, Backoff> = {};
+
 /**
  * Responsible for looking up information about a each product within
  * a `Store`. It's important that we ignore `no-await-in-loop` here
@@ -34,6 +42,14 @@ async function lookup(browser: Browser, store: Store) {
 		page.setDefaultNavigationTimeout(Config.page.navigationTimeout);
 		await page.setUserAgent(Config.page.userAgent);
 
+		if (store.disableAdBlocker) {
+			try {
+				await disableBlockerInPage(page);
+			} catch (error) {
+				Logger.error(error);
+			}
+		}
+
 		try {
 			await lookupCard(browser, store, page, link);
 		} catch (error) {
@@ -49,7 +65,41 @@ async function lookupCard(browser: Browser, store: Store, page: Page, link: Link
 	const givenWaitFor = store.waitUntil ? store.waitUntil : 'networkidle0';
 	const response: Response | null = await page.goto(link.url, {waitUntil: givenWaitFor});
 
-	if (await lookupCardInStock(store, page)) {
+	if (!response) {
+		Logger.debug(Print.noResponse(link, store, true));
+	}
+
+	let backoff = storeBackoff[store.name];
+
+	if (!backoff) {
+		backoff = {count: 0, time: Config.browser.minBackoff};
+		storeBackoff[store.name] = backoff;
+	}
+
+	if (response?.status() === 403) {
+		Logger.warn(Print.backoff(link, store, backoff.time, true));
+		await delay(backoff.time);
+		backoff.count++;
+		backoff.time = Math.min(backoff.time * 2, Config.browser.maxBackoff);
+		return;
+	}
+
+	if (response?.status() === 429) {
+		Logger.warn(Print.rateLimit(link, store, true));
+		return;
+	}
+
+	if ((response?.status() || 200) >= 400) {
+		Logger.warn(Print.badStatusCode(link, store, response!.status(), true));
+		return;
+	}
+
+	if (backoff.count > 0) {
+		backoff.count--;
+		backoff.time = Math.max(backoff.time / 2, Config.browser.minBackoff);
+	}
+
+	if (await lookupCardInStock(store, page, link)) {
 		const givenUrl = link.cartUrl ? link.cartUrl : link.url;
 		Logger.info(`${Print.inStock(link, store, true)}\n${givenUrl}`);
 
@@ -77,48 +127,48 @@ async function lookupCard(browser: Browser, store: Store, page: Page, link: Link
 			link.screenshot = `success-${Date.now()}.png`;
 			await page.screenshot({path: link.screenshot});
 		}
-
-		return;
 	}
-
-	if (await lookupPageHasCaptcha(store, page)) {
-		Logger.warn(Print.captcha(link, store, true));
-		await delay(getSleepTime());
-		return;
-	}
-
-	if (response && response.status() === 429) {
-		Logger.warn(Print.rateLimit(link, store, true));
-		return;
-	}
-
-	Logger.info(Print.outOfStock(link, store, true));
 }
 
-async function lookupCardInStock(store: Store, page: Page) {
+async function lookupCardInStock(store: Store, page: Page, link: Link) {
-	const stockHandle = await page.$(store.labels.inStock.container);
+	const baseOptions: Selector = {
+		requireVisible: false,
+		selector: store.labels.container ?? 'body',
+		type: 'textContent'
+	};
 
-	const visible = await page.evaluate(element => element && element.offsetWidth > 0 && element.offsetHeight > 0, stockHandle);
+	if (store.labels.inStock) {
-	if (!visible) {
+		const options = {...baseOptions, requireVisible: true, type: 'outerHTML' as const};
-		return false;
+
+		if (!await pageIncludesLabels(page, store.labels.inStock, options)) {
+			Logger.info(Print.outOfStock(link, store, true));
+			return false;
+		}
 	}
 
-	const stockContent = await page.evaluate(element => element.outerHTML, stockHandle);
+	if (store.labels.outOfStock) {
-
+		if (await pageIncludesLabels(page, store.labels.outOfStock, baseOptions)) {
-	Logger.debug(stockContent);
+			Logger.info(Print.outOfStock(link, store, true));
-
+			return false;
-	return includesLabels(stockContent, store.labels.inStock.text);
+		}
-}
-
-async function lookupPageHasCaptcha(store: Store, page: Page) {
-	if (!store.labels.captcha) {
-		return false;
 	}
 
-	const captchaHandle = await page.$(store.labels.captcha.container);
+	if (store.labels.bannedSeller) {
-	const captchaContent = await page.evaluate(element => element.textContent, captchaHandle);
+		if (await pageIncludesLabels(page, store.labels.bannedSeller, baseOptions)) {
+			Logger.warn(Print.bannedSeller(link, store, true));
+			return false;
+		}
+	}
 
-	return includesLabels(captchaContent, store.labels.captcha.text);
+	if (store.labels.captcha) {
+		if (await pageIncludesLabels(page, store.labels.captcha, baseOptions)) {
+			Logger.warn(Print.captcha(link, store, true));
+			await delay(getSleepTime());
+			return false;
+		}
+	}
+
+	return true;
 }
 
 export async function tryLookupAndLoop(browser: Browser, store: Store) {

src/store/model/helpers/card.ts

@@ -0,0 +1,48 @@
+export interface Card {
+	brand: string;
+	model: string;
+}
+
+export function parseCard(name: string): Card | null {
+	name = name.replace(/[^\w ]+/g, '').trim();
+	name = name.replace(/\bgraphics card\b/gi, '').trim();
+	name = name.replace(/\b\w+ fan\b/gi, '').trim();
+	name = name.replace(/\s{2,}/g, ' ');
+
+	let model = name.split(' ');
+	const brand = model.shift();
+
+	if (!brand) {
+		return null;
+	}
+
+	// Some vendors have oc at the beginning of the product name,
+	// store whether the card contains the term "oc" and remove
+	// it during filtering, then add it to the end of the name.
+	let isOC = false;
+
+	/* eslint-disable @typescript-eslint/prefer-regexp-exec */
+	model = model.filter(word => {
+		if (word.toLowerCase() === 'oc') {
+			isOC = true;
+			return false;
+		}
+
+		return !word.match(/^(nvidia|geforce|rtx|amp[ae]re|graphics|card|gpu|pci-?e(xpress)?|ray-?tracing|ray|tracing|core|boost)$/i) &&
+			!word.match(/^(\d+(?:gb?|mhz)?|gb|mhz|g?ddr(\d+x?)?)$/i);
+	});
+	/* eslint-enable @typescript-eslint/prefer-regexp-exec */
+
+	if (isOC) {
+		model.push('OC');
+	}
+
+	if (model.length === 0) {
+		return null;
+	}
+
+	return {
+		brand: brand.toLowerCase(),
+		model: model.join(' ').toLowerCase().replace(/ gaming\b/g, '').trim()
+	};
+}

src/store/model/store.ts

@@ -1,13 +1,15 @@
 import {Browser, LoadEvent} from 'puppeteer';
 
 export type Element = {
-	container: string;
+	container?: string;
 	text: string[];
 };
 
+export type Series = 'test:series' | '3070' | '3080' | '3090';
+
 export type Link = {
-	brand: 'test:brand' | 'asus' | 'evga' | 'gigabyte' | 'pny' | 'msi' | 'nvidia' | 'zotac';
+	brand: 'test:brand' | 'asus' | 'evga' | 'gigabyte' | 'inno3d' | 'kfa2' | 'palit' | 'pny' | 'msi' | 'nvidia' | 'zotac';
-	series: 'test:series' | '3070' | '3080' | '3090';
+	series: Series;
 	model: string;
 	url: string;
 	cartUrl?: string;
@@ -15,13 +17,23 @@ export type Link = {
 	screenshot?: string;
 };
 
+export type LabelQuery = Element[] | Element | string[];
+
 export type Labels = {
-	captcha?: Element;
+	bannedSeller?: LabelQuery;
-	inStock: Element;
+	captcha?: LabelQuery;
+	container?: string;
+	inStock?: LabelQuery;
+	outOfStock?: LabelQuery;
 };
 
 export type Store = {
+	disableAdBlocker?: boolean;
 	links: Link[];
+	linksBuilder?: {
+		builder: (docElement: cheerio.Cheerio, series: Series) => Link[];
+		urls: Array<{series: Series; url: string}>;
+	};
 	labels: Labels;
 	name: string;
 	setupAction?: (browser: Browser) => void;

src/util.ts

@@ -1,5 +1,6 @@
+import {Browser, Page, Response} from 'puppeteer';
 import {Config} from './config';
-import {Page} from 'puppeteer';
+import {Logger} from './logger';
 import {disableBlockerInPage} from './adblocker';
 
 export function getSleepTime() {
@@ -12,6 +13,34 @@ export async function delay(ms: number) {
 	});
 }
 
+export async function usingResponse<T>(
+	browser: Browser,
+	url: string,
+	cb: (response: (Response | null), page: Page, browser: Browser) => Promise<T>
+): Promise<T> {
+	return usingPage(browser, async (page, browser) => {
+		const response = await page.goto(url, {waitUntil: 'domcontentloaded'});
+
+		return cb(response, page, browser);
+	});
+}
+
+export async function usingPage<T>(browser: Browser, cb: (page: Page, browser: Browser) => Promise<T>): Promise<T> {
+	const page = await browser.newPage();
+	page.setDefaultNavigationTimeout(Config.page.navigationTimeout);
+	await page.setUserAgent(Config.page.userAgent);
+
+	try {
+		return await cb(page, browser);
+	} finally {
+		try {
+			await closePage(page);
+		} catch (error) {
+			Logger.error(error);
+		}
+	}
+}
+
 export async function closePage(page: Page) {
 	await disableBlockerInPage(page);
 	await page.close();

tsconfig.json

@@ -2,7 +2,7 @@
   "compilerOptions": {
     "target": "es5",
     "module": "commonjs",
-    "lib": ["es6"],
+    "lib": ["es6", "dom"],
     "allowJs": true,
     "outDir": "build",
     "rootDir": "src",