feat(lookup): use random user agents (#1335)

src/util.ts

@@ -2,6 +2,7 @@ import {Browser, Page, Response} from 'puppeteer';
 import {StatusCodeRangeArray, Store} from './store/model';
 import {config} from './config';
 import {disableBlockerInPage} from './adblocker';
+import {getRandom} from 'random-useragent';
 import {logger} from './logger';
 
 export function getSleepTime(store: Store) {
@@ -57,7 +58,7 @@ export async function usingPage<T>(
 ): Promise<T> {
 	const page = await browser.newPage();
 	page.setDefaultNavigationTimeout(config.page.timeout);
-	await page.setUserAgent(getRandomUserAgent());
+	await page.setUserAgent(await getRandomUserAgent(browser));
 
 	try {
 		return await cb(page, browser);
@@ -78,8 +79,13 @@ export async function closePage(page: Page) {
 	await page.close();
 }
 
-export function getRandomUserAgent(): string {
-	return config.page.userAgents[
-		Math.floor(Math.random() * config.page.userAgents.length)
-	];
+export async function getRandomUserAgent(browser: Browser): Promise<string> {
+	const userAgent =
+		getRandom((ua) => {
+			return ua.browserName === 'Chrome' && ua.browserVersion > '20';
+		}) ?? (await browser.userAgent());
+
+	logger.debug('user agent', userAgent);
+
+	return userAgent;
 }