I am scraping a pool of URLs with a python/selenium framework using Firefox and Geckodriver.
For each URL, all I am doing is to find one element by class name and save it to disk. In order speed this up as much as possible, I am creating a custom Firefox profile with the specs defined below in firefox_profile.set_preference.
The challenge is that testing all combinations of all Firefox profile parameters in order to know which specs are best for speed is laborious. Therefore, I am hoping to get some input.
-
Are there any preferences parameters beyond the ones mentioned below that also would speed up Firefox?
-
Is any of those preference parameters mentioned below not suitable to speed up Firefox?
def create_driver(): firefox_profile = webdriver.FirefoxProfile() firefox_profile.set_preference('browser.download.animateNotifications', False) firefox_profile.set_preference('browser.fullscreen.animate', False) firefox_profile.set_preference('browser.preferences.animateFadeIn', False) firefox_profile.set_preference('browser.tabs.animate', False) firefox_profile.set_preference('browser.cache.use_new_backend', 1) firefox_profile.set_preference('browser.sessionhistory.max_total_viewers', 0) firefox_profile.set_preference('browser.safebrowsing.enabled', False) firefox_profile.set_preference('browser.shell.checkDefaultBrowser', False) firefox_profile.set_preference('browser.startup.page', 0) firefox_profile.set_preference('layout.animated-image-layers.enabled', False) firefox_profile.set_preference('extensions.checkCompatibility', False) firefox_profile.set_preference('extensions.checkUpdateSecurity', False) firefox_profile.set_preference('extensions.logging.enabled', False) firefox_profile.set_preference('extensions.update.autoUpdateEnabled', False) firefox_profile.set_preference('extensions.update.enabled', False) firefox_profile.set_preference('print.postscript.enabled', False) firefox_profile.set_preference('toolkit.storage.synchronous', 0) firefox_profile.set_preference('image.animation_mode', 'none') firefox_profile.set_preference('images.dither', False) firefox_profile.set_preference('content.notify.interval', 1000000) firefox_profile.set_preference('content.switch.treshold', 100000) firefox_profile.set_preference('nglayout.initialpaint.delay', 1000000) firefox_profile.set_preference('network.dnscacheentries', 200) firefox_profile.set_preference('network.dnscacheexpiration', 600) firefox_profile.set_preference('network.prefetch-next', False) firefox_profile.set_preference('permissions.default.image', False) firefox_profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so', False) firefox_profile.set_preference('dom.ipc.plugins.flash.disable-protected-mode', False) firefox_profile.set_preference('app.update.enabled', False) firefox_profile.set_preference('app.update.service.enabled', False) firefox_profile.set_preference('app.update.auto', False) firefox_profile.set_preference('app.update.staging', False) firefox_profile.set_preference('app.update.silent', False) firefox_profile.set_preference("javascript.enabled", False) driver = webdriver.Firefox(firefox_profile = firefox_profile) return driver 
