feat: add proxy rotation (settable per store) (#1026)

This commit is contained in:
Mark Dietzer
2020-12-01 19:25:48 -08:00
committed by GitHub
parent 65df944973
commit 490d44e1fd
8 changed files with 498 additions and 49 deletions
-11
View File
@@ -1,25 +1,14 @@
import {startAPIServer, stopAPIServer} from './web';
import {Browser} from 'puppeteer';
import {adBlocker} from './adblocker';
import {config} from './config';
import {getSleepTime} from './util';
import {logger} from './logger';
import puppeteer from 'puppeteer-extra';
import resourceBlock from 'puppeteer-extra-plugin-block-resources';
import stealthPlugin from 'puppeteer-extra-plugin-stealth';
import {storeList} from './store/model';
import {tryLookupAndLoop} from './store';
puppeteer.use(stealthPlugin());
if (config.browser.lowBandwidth) {
puppeteer.use(
resourceBlock({
blockedTypes: new Set(['image', 'font'] as const)
})
);
} else {
puppeteer.use(adBlocker);
}
let browser: Browser | undefined;