From 5edd7906990509b6523b2397ce63e6f21c00fe32 Mon Sep 17 00:00:00 2001 From: Helium Date: Tue, 20 Jun 2023 16:37:55 -0700 Subject: [PATCH] made it compatible --- Forums/AbyssForum/crawler_selenium.py | 280 +++++++++++ Forums/AbyssForum/geckodriver.log | 462 ++++++++++++++++++ Forums/AbyssForum/parser.py | 352 +++++++++++++ Forums/Initialization/forumsList.txt | 3 +- Forums/Initialization/forums_mining.py | 6 +- Forums/Initialization/geckodriver.log | 110 +++++ Forums/OnniForums/crawler_selenium.py | 63 +-- MarketPlaces/Initialization/geckodriver.log | 75 +++ .../LionMarketplace/crawler_selenium.py | 2 +- MarketPlaces/TorBay/crawler_selenium.py | 21 +- 10 files changed, 1327 insertions(+), 47 deletions(-) create mode 100644 Forums/AbyssForum/crawler_selenium.py create mode 100644 Forums/AbyssForum/geckodriver.log create mode 100644 Forums/AbyssForum/parser.py diff --git a/Forums/AbyssForum/crawler_selenium.py b/Forums/AbyssForum/crawler_selenium.py new file mode 100644 index 0000000..7fe1184 --- /dev/null +++ b/Forums/AbyssForum/crawler_selenium.py @@ -0,0 +1,280 @@ +__author__ = 'Helium' + +''' +AbyssForum Crawler (Selenium) +''' + +from selenium import webdriver +from selenium.common.exceptions import NoSuchElementException +from selenium.webdriver.firefox.firefox_profile import FirefoxProfile +from selenium.webdriver.firefox.firefox_binary import FirefoxBinary +from selenium.webdriver.firefox.service import Service +from selenium.webdriver.common.by import By +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.support.ui import WebDriverWait +from PIL import Image + +import urllib.parse as urlparse +import os, re, time +from datetime import date +import configparser +import subprocess +from bs4 import BeautifulSoup +from Forums.Initialization.prepare_parser import new_parse +from Forums.AbyssForum.parser import abyssForum_links_parser +from Forums.Utilities.utilities import cleanHTML + +config = configparser.ConfigParser() +config.read('../../setup.ini') +counter = 1 +baseURL = 'http://qyvjopwdgjq52ehsx6paonv2ophy3p4ivfkul4svcaw6qxlzsaboyjid.onion/' + + +# Opens Tor Browser, crawls the website +def startCrawling(): + opentor() + # forumName = getForumName() + driver = getAccess() + + if driver != 'down': + try: + login(driver) + crawlForum(driver) + except Exception as e: + print(driver.current_url, e) + closetor(driver) + + # new_parse(forumName, baseURL, False) + + +# Opens Tor Browser +def opentor(): + global pid + print("Connecting Tor...") + pro = subprocess.Popen(config.get('TOR', 'firefox_binary_path')) + pid = pro.pid + time.sleep(7.5) + input('Tor Connected. Press ENTER to continue\n') + return + + +# Login using premade account credentials and do login captcha manually +def login(driver): + # wait for listing page show up (This Xpath may need to change based on different seed url) + WebDriverWait(driver, 50).until(EC.visibility_of_element_located( + (By.XPATH, '//*[@id="sn-category-3"]'))) + + +# Returns the name of the website +def getForumName(): + name = 'AbyssForum' + return name + + +# Return the link of the website +def getFixedURL(): + url = 'http://qyvjopwdgjq52ehsx6paonv2ophy3p4ivfkul4svcaw6qxlzsaboyjid.onion/' + return url + + +# Closes Tor Browser +def closetor(driver): + # global pid + # os.system("taskkill /pid " + str(pro.pid)) + # os.system("taskkill /t /f /im tor.exe") + print('Closing Tor...') + driver.close() + time.sleep(3) + return + + +# Creates FireFox 'driver' and configure its 'Profile' +# to use Tor proxy and socket +def createFFDriver(): + ff_binary = FirefoxBinary(config.get('TOR', 'firefox_binary_path')) + + ff_prof = FirefoxProfile(config.get('TOR', 'firefox_profile_path')) + ff_prof.set_preference("places.history.enabled", False) + ff_prof.set_preference("privacy.clearOnShutdown.offlineApps", True) + ff_prof.set_preference("privacy.clearOnShutdown.passwords", True) + ff_prof.set_preference("privacy.clearOnShutdown.siteSettings", True) + ff_prof.set_preference("privacy.sanitize.sanitizeOnShutdown", True) + ff_prof.set_preference("signon.rememberSignons", False) + ff_prof.set_preference("network.cookie.lifetimePolicy", 2) + ff_prof.set_preference("network.dns.disablePrefetch", True) + ff_prof.set_preference("network.http.sendRefererHeader", 0) + ff_prof.set_preference("permissions.default.image", 3) + ff_prof.set_preference("browser.download.folderList", 2) + ff_prof.set_preference("browser.download.manager.showWhenStarting", False) + ff_prof.set_preference("browser.helperApps.neverAsk.saveToDisk", "text/plain") + ff_prof.set_preference('network.proxy.type', 1) + ff_prof.set_preference("network.proxy.socks_version", 5) + ff_prof.set_preference('network.proxy.socks', '127.0.0.1') + ff_prof.set_preference('network.proxy.socks_port', 9150) + ff_prof.set_preference('network.proxy.socks_remote_dns', True) + ff_prof.set_preference("javascript.enabled", True) + ff_prof.update_preferences() + + service = Service(config.get('TOR', 'geckodriver_path')) + + driver = webdriver.Firefox(firefox_binary=ff_binary, firefox_profile=ff_prof, service=service) + + return driver + +def getAccess(): + url = getFixedURL() + driver = createFFDriver() + try: + driver.get(url) + return driver + except: + driver.close() + return 'down' + + +# Saves the crawled html page +def savePage(page, url): + cleanPage = cleanHTML(page) + filePath = getFullPathName(url) + os.makedirs(os.path.dirname(filePath), exist_ok=True) + open(filePath, 'wb').write(cleanPage.encode('utf-8')) + return + + +# Gets the full path of the page to be saved along with its appropriate file name +def getFullPathName(url): + fileName = getNameFromURL(url) + if isDescriptionLink(url): + #..\CryptBB\HTML_Pages\\ + fullPath = r'..\AbyssForum\HTML_Pages\\' + str( + "%02d" % date.today().month) + str("%02d" % date.today().day) + str( + "%04d" % date.today().year) + r'\\' + r'Description\\' + fileName + '.html' + else: + fullPath = r'..\AbyssForum\HTML_Pages\\' + str( + "%02d" % date.today().month) + str("%02d" % date.today().day) + str( + "%04d" % date.today().year) + r'\\' + r'Listing\\' + fileName + '.html' + return fullPath + + +# Creates the file name from passed URL +def getNameFromURL(url): + global counter + name = ''.join(e for e in url if e.isalnum()) + if (name == ''): + name = str(counter) + counter = counter + 1 + return name + + +def getInterestedLinks(): + links = [] + + # Hacked Database + links.append('http://qyvjopwdgjq52ehsx6paonv2ophy3p4ivfkul4svcaw6qxlzsaboyjid.onion/viewforum.php?f=26&sid=6f7add746810784861a7ec31703a3757') + # Hire a Hacker + # links.append('http://qyvjopwdgjq52ehsx6paonv2ophy3p4ivfkul4svcaw6qxlzsaboyjid.onion/viewforum.php?f=27&sid=6f7add746810784861a7ec31703a3757') + # # Hacking Tools + # links.append('http://qyvjopwdgjq52ehsx6paonv2ophy3p4ivfkul4svcaw6qxlzsaboyjid.onion/viewforum.php?f=28&sid=6f7add746810784861a7ec31703a3757') + # # Carding Forums + # links.append('http://qyvjopwdgjq52ehsx6paonv2ophy3p4ivfkul4svcaw6qxlzsaboyjid.onion/viewforum.php?f=30&sid=6f7add746810784861a7ec31703a3757') + # # Social Media Hacking + # links.append('http://qyvjopwdgjq52ehsx6paonv2ophy3p4ivfkul4svcaw6qxlzsaboyjid.onion/viewforum.php?f=32&sid=6f7add746810784861a7ec31703a3757') + # # Hacking Tutorials + # links.append('http://qyvjopwdgjq52ehsx6paonv2ophy3p4ivfkul4svcaw6qxlzsaboyjid.onion/viewforum.php?f=12&sid=6f7add746810784861a7ec31703a3757') + # # Cracking Tutorials + # links.append('http://qyvjopwdgjq52ehsx6paonv2ophy3p4ivfkul4svcaw6qxlzsaboyjid.onion/viewforum.php?f=13&sid=6f7add746810784861a7ec31703a3757') + + return links + + +def crawlForum(driver): + print("Crawling the AbyssForum forum") + + linksToCrawl = getInterestedLinks() + visited = set(linksToCrawl) + initialTime = time.time() + + i = 0 + count = 0 + while i < len(linksToCrawl): + link = linksToCrawl[i] + print('Crawling :', link) + try: + try: + driver.get(link) + except: + driver.refresh() + html = driver.page_source + savePage(html, link) + + has_next_page = True + while has_next_page: + list = topicPages(html) + for item in list: + itemURL = urlparse.urljoin(baseURL, str(item)) + try: + driver.get(itemURL) + except: + driver.refresh() + savePage(driver.page_source, item) + driver.back() + # comment out, one topic per page + break + + # comment out, go through all pages + if count == 1: + count = 0 + break + + try: + temp = driver.find_element(by=By.XPATH, value= + '/html/body/div/div[2]/div/div[3]/div') # /html/body/div/div[2]/div/div[3]/div + link = temp.find_element(by=By.CLASS_NAME, value='pagination_next').get_attribute('href') + + if link == "": + raise NoSuchElementException + try: + driver.get(link) + except: + driver.refresh() + html = driver.page_source + savePage(html, link) + count += 1 + + except NoSuchElementException: + has_next_page = False + + except Exception as e: + print(link, e) + i += 1 + + # finalTime = time.time() + # print finalTime - initialTime + + input("Crawling AbyssForum forum done sucessfully. Press ENTER to continue\n") + + +# Returns 'True' if the link is Topic link +def isDescriptionLink(url): + if 'Thread' in url: + return True + return False + + +# Returns True if the link is a listingPage link +def isListingLink(url): + if 'viewforum' in url: + return True + return False + + +# calling the parser to define the links +def topicPages(html): + soup = BeautifulSoup(html, "html.parser") + #print(soup.find('div', id="container").find('div', id="content").find('table', {"class": "tborder clear"}).find('tbody').find('tr',{"class": "inline_row"}).find('strong').text) + return abyssForum_links_parser(soup) + + +def crawler(): + startCrawling() + # print("Crawling and Parsing Abyss .... DONE!") diff --git a/Forums/AbyssForum/geckodriver.log b/Forums/AbyssForum/geckodriver.log new file mode 100644 index 0000000..d1e3763 --- /dev/null +++ b/Forums/AbyssForum/geckodriver.log @@ -0,0 +1,462 @@ +1655762162153 geckodriver INFO Listening on 127.0.0.1:50333 +1655762166434 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50334" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileqUeAAN" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655762166904 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50334/devtools/browser/fb880205-35da-44a4-83b8-a861ce7125f1 +1655762168590 Marionette INFO Listening on port 50341 +1655762169104 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655762207624 Marionette INFO Stopped listening on port 50341 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function + +###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + +1655762207923 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655762990320 geckodriver INFO Listening on 127.0.0.1:50884 +1655762994595 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50885" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileurjxEe" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655762995244 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50885/devtools/browser/36503f58-7ab7-4188-81b0-fa1b81f32c0a +1655762996997 Marionette INFO Listening on port 50890 +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +1655762997277 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655763083091 Marionette INFO Stopped listening on port 50890 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1655763083216 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655763089557 geckodriver INFO Listening on 127.0.0.1:50923 +1655763093759 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50924" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile5mY1qq" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655763094208 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50924/devtools/browser/68ce2df9-521f-4258-94f7-c2e2b199701a +1655763095918 Marionette INFO Listening on port 50929 +1655763096408 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655763143935 Marionette INFO Stopped listening on port 50929 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1655763144495 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655763166975 geckodriver INFO Listening on 127.0.0.1:50947 +1655763171175 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50948" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilenAGZM5" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655763171719 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50948/devtools/browser/0b9966bd-0cec-45cc-9d11-02a2580233f0 +1655763173307 Marionette INFO Listening on port 50953 +1655763173368 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655763208922 Marionette INFO Stopped listening on port 50953 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData] +!!! error running onStopped callback: TypeError: callback is not a function +1655763209261 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655763387409 geckodriver INFO Listening on 127.0.0.1:49752 +1655763391806 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "49753" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilevAHJSJ" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655763392827 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:49753/devtools/browser/7fe49a16-940f-4ee8-9a2f-48739a92db78 +1655763394700 Marionette INFO Listening on port 49759 +1655763395103 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655763430848 Marionette INFO Stopped listening on port 49759 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData] +!!! error running onStopped callback: TypeError: callback is not a function +1655763431145 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655765589286 geckodriver INFO Listening on 127.0.0.1:50102 +1655765621677 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50103" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile8ecUpb" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655765622149 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:50103/devtools/browser/e5a916fc-987c-43d4-9c12-9c9b88cca242 +1655765623776 Marionette INFO Listening on port 50109 +1655765623892 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655766125168 Marionette INFO Stopped listening on port 50109 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1655766125303 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655766505099 geckodriver INFO Listening on 127.0.0.1:50156 +1655766509351 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50157" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileVqvmzh" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655766509876 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50157/devtools/browser/13084c14-64d3-48a5-8b99-4c514e961d4f +1655766511328 Marionette INFO Listening on port 50162 +1655766511395 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655766552298 Marionette INFO Stopped listening on port 50162 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData] +!!! error running onStopped callback: TypeError: callback is not a function +1655766552578 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655767657126 geckodriver INFO Listening on 127.0.0.1:50257 +1655767661375 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50258" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileB9Dzeh" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655767661843 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50258/devtools/browser/867cd318-6430-4b3f-9b63-8c6bdf17636a +1655767663330 Marionette INFO Listening on port 50263 +1655767663431 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655767668263 Marionette INFO Stopped listening on port 50263 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function + +###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + +1655767668591 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655767689480 geckodriver INFO Listening on 127.0.0.1:50276 +1655767693635 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50277" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile8ZOQe7" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655767694215 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50277/devtools/browser/a70a3f5b-19af-4a68-99c8-c46086ba9599 +1655767695725 Marionette INFO Listening on port 50282 +1655767695830 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655767701272 Marionette INFO Stopped listening on port 50282 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1655767701594 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655767791360 geckodriver INFO Listening on 127.0.0.1:50296 +1655767795575 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50297" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile4GEqBr" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655767796001 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50297/devtools/browser/e2d4e57a-16ce-429d-9804-e9e391199ddf +1655767797534 Marionette INFO Listening on port 50302 +1655767797661 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655767802833 Marionette INFO Stopped listening on port 50302 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1655767803151 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655767902908 geckodriver INFO Listening on 127.0.0.1:50320 +1655767907009 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50321" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileqlJ8ZA" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655767907484 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:50321/devtools/browser/49e77669-01dd-447e-a804-36d42b3400cf +1655767908782 Marionette INFO Listening on port 50326 +1655767909069 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655767916218 Marionette INFO Stopped listening on port 50326 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1655767916523 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655767958348 geckodriver INFO Listening on 127.0.0.1:50340 +1655767962557 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50341" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilekpCdfs" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655767963032 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50341/devtools/browser/b4d1728c-d26f-4537-adc2-5c954b171b13 +1655767964591 Marionette INFO Listening on port 50346 +1655767964633 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655767973210 Marionette INFO Stopped listening on port 50346 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1655767973501 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655767998084 geckodriver INFO Listening on 127.0.0.1:50358 +1655768002314 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50359" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileujYNBj" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655768002757 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50359/devtools/browser/04112c45-204d-43f1-9615-34782fd06632 +1655768004165 Marionette INFO Listening on port 50364 +1655768004375 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655768009787 Marionette INFO Stopped listening on port 50364 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] + +###!!! [Child][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + +!!! error running onStopped callback: TypeError: callback is not a function + +###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + +1655768010100 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655768083966 geckodriver INFO Listening on 127.0.0.1:50386 +1655768088234 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50387" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile39Pl1f" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655768088664 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50387/devtools/browser/3d6cda42-cb60-4c67-8d76-1d186a51887f +1655768090263 Marionette INFO Listening on port 50392 +1655768090299 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655768091655 Marionette INFO Stopped listening on port 50392 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function + +###!!! [Parent][MessageChannel] Error: (msgtype=0x390076,name=PContent::Msg_DestroyBrowsingContextGroup) Closed channel: cannot send/recv + + +###!!! [Parent][MessageChannel] Error: (msgtype=0x390076,name=PContent::Msg_DestroyBrowsingContextGroup) Closed channel: cannot send/recv + + +###!!! [Parent][MessageChannel] Error: (msgtype=0x390076,name=PContent::Msg_DestroyBrowsingContextGroup) Closed channel: cannot send/recv + +1655768092659 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655768101201 geckodriver INFO Listening on 127.0.0.1:50404 +1655768105373 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50405" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile2YRGSz" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655768105853 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50405/devtools/browser/9c4a4d71-25ff-4980-a54d-8545d6200790 +1655768107431 Marionette INFO Listening on port 50410 +1655768107530 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655768119155 Marionette INFO Stopped listening on port 50410 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1655768119438 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655768229472 geckodriver INFO Listening on 127.0.0.1:50431 +1655768233458 webdriver::server WARN Rejected request with missing Host header diff --git a/Forums/AbyssForum/parser.py b/Forums/AbyssForum/parser.py new file mode 100644 index 0000000..49f8e37 --- /dev/null +++ b/Forums/AbyssForum/parser.py @@ -0,0 +1,352 @@ +__author__ = 'Helium' + +# Here, we are importing the auxiliary functions to clean or convert data +from Forums.Utilities.utilities import * +from datetime import date +from datetime import timedelta +import re + +# Here, we are importing BeautifulSoup to search through the HTML tree +from bs4 import BeautifulSoup + +# This is the method to parse the Description Pages (one page to each topic in the Listing Pages) + +def cryptBB_description_parser(soup): + + # Fields to be parsed + + topic = "-1" # topic name + user = [] # all users of each post + addDate = [] # all dated of each post + feedback = [] # all feedbacks of each vendor (this was found in just one Forum and with a number format) + status = [] # all user's authority in each post such as (adm, member, dangerous) + reputation = [] # all users's karma in each post (usually found as a number) + sign = [] # all user's signature in each post (usually a standard message after the content of the post) + post = [] # all messages of each post + interest = [] # all user's interest in each post + + # Finding the topic (should be just one coming from the Listing Page) + + li = soup.find("td", {"class": "thead"}).find('strong') + topic = li.text + topic = re.sub("\[\w*\]", '', topic) + + topic = topic.replace(",","") + topic = topic.replace("\n","") + topic = cleanString(topic.strip()) + print(topic) + # Finding the repeated tag that corresponds to the listing of posts + + # posts = soup.find("form", {"name": "quickModForm"}).findAll('div', {"class": "windowbg"}) + \ + # soup.find("form", {"name": "quickModForm"}).findAll('div', {"class": "windowbg2"}) + + try: + posts = soup.find('table', {"class": "tborder tfixed clear"}).find('td', {"id": "posts_container"}).find_all( + 'div', {"class": "post"}) + # print(len(posts)) + + # For each message (post), get all the fields we are interested to: + + for ipost in posts: + + # Finding a first level of the HTML page + + # post_wrapper = ipost.find('div', {"class": "post_wrapper"}).find('div', {"class": "poster"}) + post_wrapper = ipost.find('span', {"class": "largetext"}) + # Finding the author (user) of the post + + # author = post_wrapper.find('h4') + author = post_wrapper.text.strip() + # print("author " + author) + user.append(cleanString(author)) # Remember to clean the problematic characters + + # Finding the status of the author + + smalltext = ipost.find('div', {"class": "post_author"}) + + # Testing here two possibilities to find this status and combine them + if ipost.find('div', {"class": "deleted_post_author"}): + status.append(-1) + interest.append(-1) + reputation.append(-1) + addDate.append(-1) + post.append("THIS POST HAS BEEN REMOVED!") + sign.append(-1) + feedback.append(-1) + continue + + # CryptBB does have membergroup and postgroup + + membergroup = smalltext.find('div', {"class": "profile-rank"}) + postgroup = smalltext.find('div', {"class": "postgroup"}) + if membergroup != None: + membergroup = membergroup.text.strip() + if postgroup != None: + postgroup = postgroup.text.strip() + membergroup = membergroup + " - " + postgroup + else: + if postgroup != None: + membergroup = postgroup.text.strip() + else: + membergroup = "-1" + + status.append(cleanString(membergroup)) + # print("status " + cleanString(membergroup)) + # Finding the interest of the author + # CryptBB does not have blurb + blurb = smalltext.find('li', {"class": "blurb"}) + if blurb != None: + blurb = blurb.text.strip() + else: + blurb = "-1" + interest.append(cleanString(blurb)) + + # Finding the reputation of the user + # CryptBB does have reputation + author_stats = smalltext.find('div', {"class": "author_statistics"}) + karma = author_stats.find('strong') + if karma != None: + karma = karma.text + karma = karma.replace("Community Rating: ", "") + karma = karma.replace("Karma: ", "") + karma = karma.strip() + else: + karma = "-1" + reputation.append(cleanString(karma)) + # print("karma " + cleanString(karma)) + # Getting here another good tag to find the post date, post content and users' signature + + postarea = ipost.find('div', {"class": "post_content"}) + + dt = postarea.find('span', {"class": "post_date"}).text + # dt = dt.strip().split() + dt = dt.strip() + day=date.today() + if "Yesterday" in dt: + yesterday = day - timedelta(days=1) + yesterday = yesterday.strftime('%m-%d-%Y') + stime = dt.replace('Yesterday,','').strip() + date_time_obj = yesterday+ ', '+stime + date_time_obj = datetime.strptime(date_time_obj,'%m-%d-%Y, %I:%M %p') + elif "hours ago" in dt: + day = day.strftime('%m-%d-%Y') + date_time_obj = postarea.find('span', {"class": "post_date"}).find('span')['title'] + date_time_obj = datetime.strptime(date_time_obj, '%m-%d-%Y, %I:%M %p') + else: + date_time_obj = datetime.strptime(dt, '%m-%d-%Y, %I:%M %p') + stime = date_time_obj.strftime('%b %d, %Y') + sdate = date_time_obj.strftime('%I:%M %p') + + + addDate.append(date_time_obj) + # print("date " + str(date_time_obj)) + # Finding the date of the post + # date_time_obj = datetime.strptime(dt, '%a %b %d, %Y %I:%M %p') + # smalltext = postarea.find('div', {"class": "flow_hidden"}).find('div', {"class": "keyinfo"})\ + # .find('div', {"class": "smalltext"}) + # sdatetime = smalltext.text + # sdatetime = sdatetime.replace(u"\xab","") # Removing unnecessary characters + # sdatetime = sdatetime.replace(u"\xbb","") # Removing unnecessary characters + # sdatetime = sdatetime.split("on: ") # Removing unnecessary characters + # sdatetime = sdatetime[1].strip() + # stime = sdatetime[:-12:-1] # Finding the time of the post + # stime = stime[::-1] + # sdate = sdatetime.replace(stime,"") # Finding the date of the post + # sdate = sdate.replace(",","") + # sdate = sdate.strip() + + # Covert the date of the post that can be informed as: "12 February 2016", "today", "yesterday". We need + # a date format here as "mm/dd/yyyy" + + # addDate.append(convertDate(sdate,"english", crawlerDate) + " " + stime) + + # Finding the post + + inner = postarea.find('div', {"class": "post_body scaleimages"}) + inner = inner.text.strip() + # print(inner) + post.append(cleanString(inner)) + + # Finding the users's signature + + # signature = ipost.find('div', {"class": "post_wrapper"}).find('div', {"class": "moderatorbar"}).find('div', {"class": "signature"}) + signature = ipost.find('div', {"class": "signature scaleimages"}) + if signature != None: + signature = signature.text.strip() + # print(signature) + else: + signature = "-1" + sign.append(cleanString(signature)) + + # As no information about users's feedback was found, just assign "-1" to the variable + + feedback.append("-1") + except: + if soup.find('td', {"class": "trow1"}).text == " You do not have permission to access this page. ": + user.append("-1") + status.append(-1) + interest.append(-1) + reputation.append(-1) + addDate.append(-1) + post.append("NO ACCESS TO THIS PAGE!") + sign.append(-1) + feedback.append(-1) + + + # Populate the final variable (this should be a list with all fields scraped) + + row = (topic, post, user, addDate, feedback, status, reputation, sign, interest) + + # Sending the results + + return row + +# This is the method to parse the Listing Pages (one page with many posts) + +def onniForums_listing_parser(soup): + + board = "-1" # board name (the previous level of the topic in the Forum categorization tree. + # For instance: Security/Malware/Tools to hack Facebook. The board here should be Malware) + + nm = 0 # this variable should receive the number of topics + topic = [] # all topics + user = [] # all users of each topic + post = [] # number of posts of each topic + view = [] # number of views of each topic + addDate = [] # when the topic was created (difficult to find) + href = [] # this variable should receive all cleaned urls (we will use this to do the marge between + # Listing and Description pages) + + # Finding the board (should be just one) + + board = soup.find('span', {"class": "active"}).text + board = cleanString(board.strip()) + + # Finding the repeated tag that corresponds to the listing of topics + + itopics = soup.find_all('tr',{"class": "inline_row"}) + index = 0 + for itopic in itopics: + + # For each topic found, the structure to get the rest of the information can be of two types. Testing all of them + # to don't miss any topic + + + # Adding the topic to the topic list + try: + topics = itopic.find('span', {"class": "subject_old"}).find('a').text + except: + topics = itopic.find('span', {"class": "subject_new"}).find('a').text + topics = re.sub("\[\w*\]", '', topics) + topic.append(cleanString(topics)) + + # Counting how many topics we have found so far + + nm = len(topic) + + # Adding the url to the list of urls + try: + link = itopic.find('span', {"class": "subject_old"}).find('a').get('href') + except: + link = itopic.find('span',{"class": "subject_new"}).find('a').get('href') + link = cleanLink(link) + href.append(link) + + # Finding the author of the topic + ps = itopic.find('div', {"class":"author smalltext"}).find('a').text + author = ps.strip() + user.append(cleanString(author)) + + # Finding the number of replies + columns = itopic.findChildren('td',recursive=False) + posts = columns[3].text + + post.append(cleanString(posts)) + + # Finding the number of Views + tview = columns[4].text + view.append(cleanString(tview)) + + + + # If no information about when the topic was added, just assign "-1" to the variable + #dt = itopic.find('div', {"class": "responsive-hide"}).text.split('ยป')[1] + #dt = dt.strip() + #date_time_obj = datetime.strptime(dt,'%a %b %d, %Y %I:%M %p') + #addDate.append(date_time_obj) + addDate.append("-1") + + + + index += 1 + return organizeTopics("CryptBB", nm, topic, board, view, post, user, addDate, href) + + # if len(tag) > 0: + # + # # Finding the topic + # + # tds = tds[0].find(tag[0]) + # topics = tds.text + # topics = topics.replace(u"\xbb","") + # topics = topics.strip() + # topic.append(cleanString(topics)) + # + # # Counting how many topics we have found so far + # + # nm = len(topic) + # + # # Adding the url to the list of urls + # + # link = tds.findAll('a', href=True) + # link = link[0].get('href') + # link = cleanLink(link) + # href.append(link) + # + # # Finding the author of the topic + # + # ps = itopic.find('td', {"class": tag[1]}).find('p').find('a') + # if ps == None: + # ps = itopic.find('td', {"class": tag[1]}).find('p') + # ps = ps.text.replace("Started by ","") + # else: + # ps = ps.text + # author = ps.strip() + # user.append(cleanString(author)) + # + # # Finding the number of replies + # + # statistics = itopic.find('td', {"class": tag[2]}) + # statistics = statistics.text + # statistics = statistics.split("Replies") + # posts = statistics[0].strip() + # post.append(cleanString(posts)) + # + # # Finding the number of Views + # + # views = statistics[1] + # views = views.replace("Views","") + # views = views.strip() + # view.append(cleanString(views)) + # + # # As no information about when the topic was added, just assign "-1" to the variable + # + # addDate.append("-1") + + #return organizeTopics("TheMajesticGarden", nm, topic, board, view, post, user, addDate, href) + +#need to change this method +def abyssForum_links_parser(soup): + + # Returning all links that should be visited by the Crawler + + href = [] + #print(soup.find('table', {"class": "tborder clear"}).find( + # 'tbody').find_all('tr', {"class": "inline_row"})) + listing = soup.find_all('tr', {"class": "list-inner"}) + + for a in listing: + link = a.find('a', {"class": "topictitle"}).get('href') + + href.append(link) + + return href \ No newline at end of file diff --git a/Forums/Initialization/forumsList.txt b/Forums/Initialization/forumsList.txt index ebdc9b3..cdfc2c4 100644 --- a/Forums/Initialization/forumsList.txt +++ b/Forums/Initialization/forumsList.txt @@ -1 +1,2 @@ -CryptBB \ No newline at end of file +OnniForums +AbyssForum \ No newline at end of file diff --git a/Forums/Initialization/forums_mining.py b/Forums/Initialization/forums_mining.py index f431f97..317e0ac 100644 --- a/Forums/Initialization/forums_mining.py +++ b/Forums/Initialization/forums_mining.py @@ -9,7 +9,7 @@ from datetime import * from Forums.BestCardingWorld.crawler_selenium import crawler as crawlerBestCardingWorld from Forums.CryptBB.crawler_selenium import crawler as crawlerCryptBB from Forums.OnniForums.crawler_selenium import crawler as crawlerOnniForums -# from Forums.CrackingPro.crawler_selenium import crawler as crawlerCrackingPro +from Forums.AbyssForum.crawler_selenium import crawler as crawlerAbyssForum import time @@ -100,8 +100,8 @@ if __name__ == '__main__': crawlerCryptBB() elif forum == "OnniForums": crawlerOnniForums() - elif forum == "CrackingPro": - crawlerCrackingPro() + elif forum == "AbyssForum": + crawlerAbyssForum() print("Scraping process completed successfully!") diff --git a/Forums/Initialization/geckodriver.log b/Forums/Initialization/geckodriver.log index d46de81..622aee0 100644 --- a/Forums/Initialization/geckodriver.log +++ b/Forums/Initialization/geckodriver.log @@ -5521,3 +5521,113 @@ unwatchForTargets()@TargetList.jsm:37 destructor()@TargetList.jsm:109 stop()@CDP.jsm:104 close()@RemoteAgent.jsm:138 +1687302747101 geckodriver INFO Listening on 127.0.0.1:51672 +1687302750552 mozrunner::runner INFO Running command: "C:\\Users\\\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "51673" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofilecHL7jg" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: init()" +console.log: "TorConnect: Entering Initial state" +console.log: "TorConnect: Observed profile-after-change" +console.log: "TorConnect: Observing topic 'TorProcessExited'" +console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: Observing topic 'torsettings:ready'" +console.log: "TorSettings: Observed profile-after-change" +1687302751101 Marionette INFO Marionette enabled +console.log: "TorConnect: Will load after bootstrap => [about:blank]" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:51673/devtools/browser/b57c1e2f-db68-4f8f-a717-c0526d8e0ab4 +1687302752687 Marionette INFO Listening on port 51678 +1687302752793 RemoteAgent WARN TLS certificate errors will be ignored for this session +1687302753088 Marionette INFO Stopped listening on port 51678 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData] +!!! error running onStopped callback: TypeError: callback is not a function +JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first. +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofilecHL7jg\thumbnails) because it does not exist +1687302753643 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1687303705696 geckodriver INFO Listening on 127.0.0.1:51763 +1687303709525 mozrunner::runner INFO Running command: "C:\\Users\\\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "51764" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofilepYUwQd" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: init()" +console.log: "TorConnect: Entering Initial state" +console.log: "TorConnect: Observed profile-after-change" +console.log: "TorConnect: Observing topic 'TorProcessExited'" +console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: Observing topic 'torsettings:ready'" +console.log: "TorSettings: Observed profile-after-change" +1687303710396 Marionette INFO Marionette enabled +console.log: "TorConnect: Will load after bootstrap => [about:blank]" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:51764/devtools/browser/c8cc38c7-0e96-4597-9055-324bd81e9669 +1687303712490 Marionette INFO Listening on port 51769 +1687303712829 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: https://onniforums.com/jscripts/thread.js?ver=1827, line 188: ReferenceError: spinner is not defined +JavaScript error: https://onniforums.com/jscripts/thread.js?ver=1827, line 188: ReferenceError: spinner is not defined +JavaScript error: resource://gre/modules/ExtensionTelemetry.jsm, line 109: Error: TelemetryStopwatch: key "WEBEXT_CONTENT_SCRIPT_INJECTION_MS" was already initialized +JavaScript error: resource://gre/modules/ExtensionTelemetry.jsm, line 113: Error: TelemetryStopwatch: key "WEBEXT_CONTENT_SCRIPT_INJECTION_MS_BY_ADDONID" was already initialized +JavaScript error: resource://gre/modules/ExtensionTelemetry.jsm, line 109: Error: TelemetryStopwatch: finishing nonexisting stopwatch. Histogram: "WEBEXT_CONTENT_SCRIPT_INJECTION_MS", key: "" +JavaScript error: resource://gre/modules/ExtensionTelemetry.jsm, line 113: Error: TelemetryStopwatch: finishing nonexisting stopwatch. Histogram: "WEBEXT_CONTENT_SCRIPT_INJECTION_MS_BY_ADDONID", key: "{73a6fe31-595d-460b-a920-fcc0f8843232}" +JavaScript error: resource://gre/modules/ExtensionTelemetry.jsm, line 109: Error: TelemetryStopwatch: finishing nonexisting stopwatch. Histogram: "WEBEXT_CONTENT_SCRIPT_INJECTION_MS", key: "" +JavaScript error: resource://gre/modules/ExtensionTelemetry.jsm, line 113: Error: TelemetryStopwatch: finishing nonexisting stopwatch. Histogram: "WEBEXT_CONTENT_SCRIPT_INJECTION_MS_BY_ADDONID", key: "{73a6fe31-595d-460b-a920-fcc0f8843232}" +1687304068063 Marionette INFO Stopped listening on port 51769 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData] +!!! error running onStopped callback: TypeError: callback is not a function +JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first. +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofilepYUwQd\thumbnails) because it does not exist + +###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + +1687304068282 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1687304215361 geckodriver INFO Listening on 127.0.0.1:51869 +1687304219693 mozrunner::runner INFO Running command: "C:\\Users\\\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "51870" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofilev3eZVX" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: init()" +console.log: "TorConnect: Entering Initial state" +console.log: "TorConnect: Observed profile-after-change" +console.log: "TorConnect: Observing topic 'TorProcessExited'" +console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: Observing topic 'torsettings:ready'" +console.log: "TorSettings: Observed profile-after-change" +1687304220438 Marionette INFO Marionette enabled +console.log: "TorConnect: Will load after bootstrap => [about:blank]" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:51870/devtools/browser/66d823f9-8386-4689-bf63-0e082efcfcc9 +1687304222794 Marionette INFO Listening on port 51875 +1687304223368 RemoteAgent WARN TLS certificate errors will be ignored for this session +Marionette threw an error: TypeError: this.curBrowser.contentBrowser is null +GeckoDriver.prototype.navigateTo@chrome://remote/content/marionette/driver.js:962:3 + + +###!!! [Child][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + +JavaScript error: resource://gre/modules/ExtensionTelemetry.jsm, line 109: Error: TelemetryStopwatch: key "WEBEXT_CONTENT_SCRIPT_INJECTION_MS" was already initialized +JavaScript error: resource://gre/modules/ExtensionTelemetry.jsm, line 113: Error: TelemetryStopwatch: key "WEBEXT_CONTENT_SCRIPT_INJECTION_MS_BY_ADDONID" was already initialized +JavaScript error: resource://gre/modules/ExtensionTelemetry.jsm, line 109: Error: TelemetryStopwatch: finishing nonexisting stopwatch. Histogram: "WEBEXT_CONTENT_SCRIPT_INJECTION_MS", key: "" +JavaScript error: resource://gre/modules/ExtensionTelemetry.jsm, line 113: Error: TelemetryStopwatch: finishing nonexisting stopwatch. Histogram: "WEBEXT_CONTENT_SCRIPT_INJECTION_MS_BY_ADDONID", key: "{73a6fe31-595d-460b-a920-fcc0f8843232}" +JavaScript error: resource://gre/modules/ExtensionTelemetry.jsm, line 109: Error: TelemetryStopwatch: finishing nonexisting stopwatch. Histogram: "WEBEXT_CONTENT_SCRIPT_INJECTION_MS", key: "" +JavaScript error: resource://gre/modules/ExtensionTelemetry.jsm, line 113: Error: TelemetryStopwatch: finishing nonexisting stopwatch. Histogram: "WEBEXT_CONTENT_SCRIPT_INJECTION_MS_BY_ADDONID", key: "{73a6fe31-595d-460b-a920-fcc0f8843232}" diff --git a/Forums/OnniForums/crawler_selenium.py b/Forums/OnniForums/crawler_selenium.py index 64fa22f..c7d669f 100644 --- a/Forums/OnniForums/crawler_selenium.py +++ b/Forums/OnniForums/crawler_selenium.py @@ -16,6 +16,7 @@ from PIL import Image import urllib.parse as urlparse import os, re, time +import configparser from datetime import date import subprocess from bs4 import BeautifulSoup @@ -23,6 +24,8 @@ from Forums.Initialization.prepare_parser import new_parse from Forums.OnniForums.parser import onniForums_links_parser from Forums.Utilities.utilities import cleanHTML +config = configparser.ConfigParser() +config.read('../../setup.ini') counter = 1 baseURL = 'http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/' @@ -48,8 +51,7 @@ def startCrawling(): def opentor(): global pid print("Connecting Tor...") - path = open('../../path.txt').readline().strip() - pro = subprocess.Popen(path) + pro = subprocess.Popen(config.get('TOR', 'firefox_binary_path')) pid = pro.pid time.sleep(7.5) input('Tor Connected. Press ENTER to continue\n') @@ -72,7 +74,8 @@ def login(driver): #Password here passwordBox.send_keys('$ourP@tchK1ds') - input("Press ENTER when log in is completed\n") + clicker = driver.find_element(by=By.XPATH, value='/html/body/div/div[2]/div/form/div/input') + clicker.click() # wait for listing page show up (This Xpath may need to change based on different seed url) WebDriverWait(driver, 50).until(EC.visibility_of_element_located( @@ -105,12 +108,9 @@ def closetor(driver): # Creates FireFox 'driver' and configure its 'Profile' # to use Tor proxy and socket def createFFDriver(): - file = open('../../path.txt', 'r') - lines = file.readlines() + ff_binary = FirefoxBinary(config.get('TOR', 'firefox_binary_path')) - ff_binary = FirefoxBinary(lines[0].strip()) - - ff_prof = FirefoxProfile(lines[1].strip()) + ff_prof = FirefoxProfile(config.get('TOR', 'firefox_profile_path')) ff_prof.set_preference("places.history.enabled", False) ff_prof.set_preference("privacy.clearOnShutdown.offlineApps", True) ff_prof.set_preference("privacy.clearOnShutdown.passwords", True) @@ -118,7 +118,7 @@ def createFFDriver(): ff_prof.set_preference("privacy.sanitize.sanitizeOnShutdown", True) ff_prof.set_preference("signon.rememberSignons", False) ff_prof.set_preference("network.cookie.lifetimePolicy", 2) - ff_prof.set_preference("network.dns.disablePrefetch", True)# + ff_prof.set_preference("network.dns.disablePrefetch", True) ff_prof.set_preference("network.http.sendRefererHeader", 0) ff_prof.set_preference("permissions.default.image", 3) ff_prof.set_preference("browser.download.folderList", 2) @@ -132,10 +132,11 @@ def createFFDriver(): ff_prof.set_preference("javascript.enabled", True) ff_prof.update_preferences() - service = Service(lines[2].strip()) + service = Service(config.get('TOR', 'geckodriver_path')) driver = webdriver.Firefox(firefox_binary=ff_binary, firefox_profile=ff_prof, service=service) + return driver @@ -190,23 +191,23 @@ def getInterestedLinks(): # Hacking & Cracking tutorials links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Hacking-Cracking-tutorials') # Hacking & Cracking questions - links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Hacking-Cracking-questions') - # Exploit PoCs - links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Exploit-PoCs') - # Cracked software - links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Cracked-software') - # Malware-development - links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Malware-development') - # Carding & Fraud - links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Carding-Fraud') - # Darknet Discussions - links.append('http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/forumdisplay.php?fid=88') - # OPSEC - links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-OPSEC') - # Databases - links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Databases') - # Proxies - links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Proxies') + # links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Hacking-Cracking-questions') + # # Exploit PoCs + # links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Exploit-PoCs') + # # Cracked software + # links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Cracked-software') + # # Malware-development + # links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Malware-development') + # # Carding & Fraud + # links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Carding-Fraud') + # # Darknet Discussions + # links.append('http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/forumdisplay.php?fid=88') + # # OPSEC + # links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-OPSEC') + # # Databases + # links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Databases') + # # Proxies + # links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Proxies') return links @@ -243,12 +244,12 @@ def crawlForum(driver): savePage(driver.page_source, item) driver.back() # comment out, one topic per page - # break + break # comment out, go through all pages - #if count == 1: - # count = 0 - # break + if count == 1: + count = 0 + break try: temp = driver.find_element(by=By.XPATH, value= diff --git a/MarketPlaces/Initialization/geckodriver.log b/MarketPlaces/Initialization/geckodriver.log index 923b779..8f67c0d 100644 --- a/MarketPlaces/Initialization/geckodriver.log +++ b/MarketPlaces/Initialization/geckodriver.log @@ -8387,3 +8387,78 @@ DevTools listening on ws://localhost:51470/devtools/browser/d447d909-9a98-4af6-a 1687301287194 RemoteAgent WARN TLS certificate errors will be ignored for this session JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1687301543621 Marionette INFO Stopped listening on port 51475 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData] +!!! error running onStopped callback: TypeError: callback is not a function +JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first. +JavaScript error: resource://gre/modules/PageThumbs.jsm, line 709: AbortError: IOUtils.profileBeforeChange getter: IOUtils: profileBeforeChange phase has already finished +[Parent 4348, IPC I/O Parent] WARNING: pipe error: 232: file /var/tmp/build/firefox-b6010b1466c9/ipc/chromium/src/chrome/common/ipc_channel_win.cc:544 +[Parent 4348, IPC I/O Parent] WARNING: file /var/tmp/build/firefox-b6010b1466c9/ipc/chromium/src/base/process_util_win.cc:167 + +###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + +1687301544107 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1687302643052 geckodriver INFO Listening on 127.0.0.1:51620 +1687302646756 mozrunner::runner INFO Running command: "C:\\Users\\\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "51621" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofileNgV5GZ" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: init()" +console.log: "TorConnect: Entering Initial state" +console.log: "TorConnect: Observed profile-after-change" +console.log: "TorConnect: Observing topic 'TorProcessExited'" +console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: Observing topic 'torsettings:ready'" +console.log: "TorSettings: Observed profile-after-change" +1687302647470 Marionette INFO Marionette enabled +console.log: "TorConnect: Will load after bootstrap => [about:blank]" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:51621/devtools/browser/714e73cc-c4b1-4916-8919-2c187790cdd6 +1687302649099 Marionette INFO Listening on port 51626 +1687302649280 RemoteAgent WARN TLS certificate errors will be ignored for this session +1687302658343 Marionette INFO Stopped listening on port 51626 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData] +!!! error running onStopped callback: TypeError: callback is not a function +JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first. +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofileNgV5GZ\thumbnails) because it does not exist +JavaScript error: chrome://remote/content/marionette/driver.js, line 2326: NS_ERROR_ILLEGAL_VALUE: Component returned failure code: 0x80070057 (NS_ERROR_ILLEGAL_VALUE) [nsIObserverService.removeObserver] + +###!!! [Child][MessageChannel] Error: (msgtype=0x590008,name=PHttpChannel::Msg_DeletingChannel) Channel closing: too late to send/recv, messages will be lost + + +###!!! [Child][MessageChannel] Error: (msgtype=0x23002E,name=PBrowser::Msg___delete__) Channel closing: too late to send/recv, messages will be lost + + +###!!! [Parent][MessageChannel] Error: (msgtype=0x390076,name=PContent::Msg_DestroyBrowsingContextGroup) Closed channel: cannot send/recv + + +###!!! [Child][MessageChannel] Error: (msgtype=0x390140,name=PContent::Msg_ScriptError) Channel closing: too late to send/recv, messages will be lost + + +###!!! [Child][MessageChannel] Error: (msgtype=0x390140,name=PContent::Msg_ScriptError) Channel closing: too late to send/recv, messages will be lost + + +###!!! [Child][MessageChannel] Error: (msgtype=0x690014,name=PNecko::Msg_PredLearn) Channel closing: too late to send/recv, messages will be lost + + +###!!! [Child][MessageChannel] Error: (msgtype=0x390140,name=PContent::Msg_ScriptError) Channel closing: too late to send/recv, messages will be lost + + +###!!! [Child][MessageChannel] Error: (msgtype=0x390140,name=PContent::Msg_ScriptError) Channel closing: too late to send/recv, messages will be lost + +1687302659162 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 diff --git a/MarketPlaces/LionMarketplace/crawler_selenium.py b/MarketPlaces/LionMarketplace/crawler_selenium.py index 0456077..0010462 100644 --- a/MarketPlaces/LionMarketplace/crawler_selenium.py +++ b/MarketPlaces/LionMarketplace/crawler_selenium.py @@ -196,7 +196,7 @@ def getInterestedLinks(): # # Spamming and Anti-Captcha # links.append('http://lionznqc2hg2wsp5vgruqait4cpknihwlje6hkjyi52lcl5ivyf7bcad.onion/category/f08a9380-c7e7-11ec-918c-ffef7c670c97') # hacked accounts - links.append('http://lionznqc2hg2wsp5vgruqait4cpknihwlje6hkjyi52lcl5ivyf7bcad.onion/category/fd47b4a0-c7e7-11ec-937b-61246c4b12b3') + #links.append('http://lionznqc2hg2wsp5vgruqait4cpknihwlje6hkjyi52lcl5ivyf7bcad.onion/category/fd47b4a0-c7e7-11ec-937b-61246c4b12b3') return links diff --git a/MarketPlaces/TorBay/crawler_selenium.py b/MarketPlaces/TorBay/crawler_selenium.py index 3c06ca3..3c0619e 100644 --- a/MarketPlaces/TorBay/crawler_selenium.py +++ b/MarketPlaces/TorBay/crawler_selenium.py @@ -18,11 +18,15 @@ import urllib.parse as urlparse import os, time from datetime import date import subprocess +import configparser +import subprocess from bs4 import BeautifulSoup from MarketPlaces.Initialization.prepare_parser import new_parse from MarketPlaces.TorBay.parser import torbay_links_parser from MarketPlaces.Utilities.utilities import cleanHTML +config = configparser.ConfigParser() +config.read('../../setup.ini') counter = 1 baseURL = 'http://torbay3253zck4ym5cbowwvrbfjjzruzthrx3np5y6owvifrnhy5ybid.onion/' @@ -50,8 +54,7 @@ def startCrawling(): def opentor(): global pid print("Connecting Tor...") - path = open('../../path.txt').readline().strip() - pro = subprocess.Popen(path) + pro = subprocess.Popen(config.get('TOR', 'firefox_binary_path')) pid = pro.pid time.sleep(7.5) input('Tor Connected. Press ENTER to continue\n') @@ -87,12 +90,9 @@ def closetor(driver): # Creates FireFox 'driver' and configure its 'Profile' # to use Tor proxy and socket def createFFDriver(): - file = open('../../path.txt', 'r') - lines = file.readlines() - - ff_binary = FirefoxBinary(lines[0].strip()) + ff_binary = FirefoxBinary(config.get('TOR', 'firefox_binary_path')) - ff_prof = FirefoxProfile(lines[1].strip()) + ff_prof = FirefoxProfile(config.get('TOR', 'firefox_profile_path')) ff_prof.set_preference("places.history.enabled", False) ff_prof.set_preference("privacy.clearOnShutdown.offlineApps", True) ff_prof.set_preference("privacy.clearOnShutdown.passwords", True) @@ -102,7 +102,7 @@ def createFFDriver(): ff_prof.set_preference("network.cookie.lifetimePolicy", 2) ff_prof.set_preference("network.dns.disablePrefetch", True) ff_prof.set_preference("network.http.sendRefererHeader", 0) - ff_prof.set_preference("permissions.default.image", 2) + ff_prof.set_preference("permissions.default.image", 3) ff_prof.set_preference("browser.download.folderList", 2) ff_prof.set_preference("browser.download.manager.showWhenStarting", False) ff_prof.set_preference("browser.helperApps.neverAsk.saveToDisk", "text/plain") @@ -111,16 +111,15 @@ def createFFDriver(): ff_prof.set_preference('network.proxy.socks', '127.0.0.1') ff_prof.set_preference('network.proxy.socks_port', 9150) ff_prof.set_preference('network.proxy.socks_remote_dns', True) - ff_prof.set_preference("javascript.enabled", False) + ff_prof.set_preference("javascript.enabled", True) ff_prof.update_preferences() - service = Service(lines[2].strip()) + service = Service(config.get('TOR', 'geckodriver_path')) driver = webdriver.Firefox(firefox_binary=ff_binary, firefox_profile=ff_prof, service=service) return driver - #the driver 'gets' the url, attempting to get on the site, if it can't access return 'down' #return: return the selenium driver or string 'down' def getAccess():