From ca8abc119ab4b30976f480bb44a76ffa8e05f96a Mon Sep 17 00:00:00 2001 From: Helium Date: Tue, 13 Jun 2023 13:36:35 -0700 Subject: [PATCH] tested onni crawler --- Forums/CrackingPro/crawler_mechanize.py | 257 +++++++++++++ Forums/CrackingPro/crawler_selenium.py | 304 ++++++++++++++++ Forums/CrackingPro/geckodriver.log | 462 ++++++++++++++++++++++++ Forums/CrackingPro/parser.py | 354 ++++++++++++++++++ Forums/CryptBB/crawler_mechanize.py | 1 - Forums/Initialization/geckodriver.log | 35 ++ Forums/OnniForums/crawler_mechanize.py | 16 +- 7 files changed, 1419 insertions(+), 10 deletions(-) create mode 100644 Forums/CrackingPro/crawler_mechanize.py create mode 100644 Forums/CrackingPro/crawler_selenium.py create mode 100644 Forums/CrackingPro/geckodriver.log create mode 100644 Forums/CrackingPro/parser.py diff --git a/Forums/CrackingPro/crawler_mechanize.py b/Forums/CrackingPro/crawler_mechanize.py new file mode 100644 index 0000000..9b12223 --- /dev/null +++ b/Forums/CrackingPro/crawler_mechanize.py @@ -0,0 +1,257 @@ +__author__ = '91Shadows' + +''' +OnniForums Crawler (Mechanize) +''' + +import codecs, os, re +import socks, socket, time +from datetime import date + +import urllib.parse as urlparse +import http.client as httplib +import mechanize +import subprocess +from bs4 import BeautifulSoup +from Forums.Initialization.prepare_parser import new_parse +from Forums.OnniForums.parser import onniForums_listing_parser + +counter = 1 +httplib.HTTPConnection._http_vsn = 10 +httplib.HTTPConnection._http_vsn_str = 'HTTP/1.0' +baseURL = 'http://bestteermb42clir6ux7xm76d4jjodh3fpahjqgbddbmfrgp4skg2wqd.onion/viewforum.php?f=42&sid=ee2cbfd73c12923d979790b2bb4bdfd5' +socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, "127.0.0.1", 9150) + + +# Opens Tor Browser, crawls the website +def startCrawling(): + opentor() + getUrl() + forumName = getForumName() + br = getAccess() + + if br != 'down': + crawlForum(br) + new_parse(forumName, False) + + # new_parse(forumName, False) + + closetor() + + +# Opens Tor Browser +def opentor(): + global pid + print("Connecting Tor...") + path = open('../../path.txt').readline() + pro = subprocess.Popen(path) + pid = pro.pid + time.sleep(7.5) + input("Tor Connected. Press ENTER to continue\n") + return + + +# Creates a connection through Tor Port +def getUrl(timeout=None): + socket.socket = socks.socksocket + socket.create_connection = create_connection + return + + +# Makes the onion address request +def create_connection(address, timeout=None, source_address=None): + sock = socks.socksocket() + sock.connect(address) + return sock + + +# Returns the name of website +def getForumName(): + name = 'CrackingPro' + return name + + +# Return the link of website +def getFixedURL(): + url = 'http://bestteermb42clir6ux7xm76d4jjodh3fpahjqgbddbmfrgp4skg2wqd.onion/viewforum.php?f=42&sid=ee2cbfd73c12923d979790b2bb4bdfd5' + + return url + + +# Closes Tor Browser +def closetor(): + global pid + os.system("taskkill /pid " + str(pid)) + print('Closing Tor...') + time.sleep(3) + return + + +# Creates a Mechanize browser and initializes its options +def createBrowser(): + br = mechanize.Browser() + cj = mechanize.CookieJar() + br.set_cookiejar(cj) + + # Browser options + br.set_handle_equiv(True) + br.set_handle_redirect(True) + br.set_handle_referer(True) + br.set_handle_robots(False) + br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1) + br.addheaders = [('User-agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko'), + ('Accept', '*/*')] + + return br + + +def getAccess(): + url = getFixedURL() + br = createBrowser() + + try: + + br.open(url) + return br + + except: + + return 'down' + + +# Saves the crawled html page +def savePage(page, url): + filePath = getFullPathName(url) + os.makedirs(os.path.dirname(filePath), exist_ok=True) + a = page.read() + open(filePath, "wb").write(a) + return + + +# Gets the full path of the page to be saved along with its appropriate file name +def getFullPathName(url): + fileName = getNameFromURL(url) + if isDescriptionLink(url): + fullPath = 'C:/Users/CALSysLab/Documents/threatIntelligence-main/DarkWebMining_Working/Forums/CrackingPro/HTML_Pages/' + str( + "%02d" % date.today().month) + str("%02d" % date.today().day) + str( + "%04d" % date.today().year) + '/' + 'Description/' + fileName + '.html' + else: + fullPath = 'C:/Users/CALSysLab/Documents/threatIntelligence-main/DarkWebMining_Working/Forums/CrackingPro/HTML_Pages/' + str( + "%02d" % date.today().month) + str("%02d" % date.today().day) + str( + "%04d" % date.today().year) + '/' + 'Listing/' + fileName + '.html' + return fullPath + + +# Creates the name of the file based on URL +def getNameFromURL(url): + global counter + name = ''.join(e for e in url if e.isalnum()) + if (name == ''): + name = str(counter) + counter = counter + 1 + return name + + +# Hacking and Markets related topics +def getInterestedLinks(): + links = [] + + links.append('http://bestteermb42clir6ux7xm76d4jjodh3fpahjqgbddbmfrgp4skg2wqd.onion/viewforum.php?f=43&sid=e12864ffccc5df877b03b573534955be') + + return links + + +# Start crawling Forum pages +def crawlForum(br): + print("Crawling CrackingPro forum") + + linksToCrawl = getInterestedLinks() + visited = set(linksToCrawl) + initialTime = time.time() + + + i = 0 + while i < len(linksToCrawl): + link = linksToCrawl[i] + print('Crawling :', link) + try: + page = br.open(link) + savePage(page, link) + + res = br.response().read() + soup = BeautifulSoup(res, 'html.parser') + + next_link = soup.find("a", {"rel": "next"}) + if next_link != None: + full_url = urlparse.urljoin(linksToCrawl[i], next_link['href']) + linksToCrawl.insert(i + 1, full_url) + + listOfTopics = findDescriptionPages(link) + for topic in listOfTopics: + itemPage = br.open(str(topic)) + savePage(itemPage, topic) + + except Exception as e: + print('Error getting link: ', link, e) + i += 1 + + # finalTime = time.time() + # print finalTime - initialTime + + input("Crawling Cracking Forum forum done sucessfully. Press ENTER to continue\n") + + return + + +# Returns True if the link is 'Topic' Links +def isDescriptionLink(url): + if 'Topic' in url: + return True + return False + + +# Returns True if the link is a listingPage link +def isListingLink(url): + ''' + reg = 'board=[0-9]+.[0-9]+\Z' + if len(re.findall(reg, url)) == 0: + return False + return True + ''' + if 'Forum' in url: + return True + return False + + +# calling the parser to define the links +def findDescriptionPages(url): + soup = "" + + error = False + try: + html = codecs.open( + "C:\\Users\\CALSysLab\\Documents\\threatIntelligence-main\\DarkWebMining_Working\\Forums\\CrackingPro\\HTML_Pages\\" + str( + "%02d" % date.today().month) + str("%02d" % date.today().day) + str( + "%04d" % date.today().year) + "\\Listing\\" + getNameFromURL(url) + ".html", encoding='utf8') + soup = BeautifulSoup(html, "html.parser") + except: + try: + html = open( + "C:\\Users\\CALSysLab\\Documents\\threatIntelligence-main\\DarkWebMining_Working\\Forums\\CrackingPro\\HTML_Pages\\" + str( + "%02d" % date.today().month) + str("%02d" % date.today().day) + str( + "%04d" % date.today().year) + "\\Listing\\" + getNameFromURL(url) + ".html") + soup = BeautifulSoup(html, "html.parser") + except: + error = True + print("There was a problem to read the file " + getNameFromURL(url) + " in the listing section.") + + if not error: + return onniForums_listing_parser(soup) + + else: + return [] + + +def crawler(): + startCrawling() + print("Crawling and Parsing CrackingPro .... DONE!") diff --git a/Forums/CrackingPro/crawler_selenium.py b/Forums/CrackingPro/crawler_selenium.py new file mode 100644 index 0000000..475f21f --- /dev/null +++ b/Forums/CrackingPro/crawler_selenium.py @@ -0,0 +1,304 @@ +__author__ = 'DarkWeb' + +''' +OnniForums Crawler (Selenium) +''' + +from selenium import webdriver +from selenium.common.exceptions import NoSuchElementException +from selenium.webdriver.firefox.firefox_profile import FirefoxProfile +from selenium.webdriver.firefox.firefox_binary import FirefoxBinary +from selenium.webdriver.firefox.service import Service +from selenium.webdriver.common.by import By +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.support.ui import WebDriverWait +from PIL import Image + +import urllib.parse as urlparse +import os, re, time +from datetime import date +import subprocess +from bs4 import BeautifulSoup +from Forums.Initialization.prepare_parser import new_parse +from Forums.OnniForums.parser import cryptBB_links_parser +from Forums.Utilities.utilities import cleanHTML + +counter = 1 +baseURL = 'http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/' + + +# Opens Tor Browser, crawls the website +def startCrawling(): + opentor() + # forumName = getForumName() + driver = getAccess() + + if driver != 'down': + try: + login(driver) + crawlForum(driver) + except Exception as e: + print(driver.current_url, e) + closetor(driver) + + # new_parse(forumName, False) + + +# Opens Tor Browser +def opentor(): + global pid + print("Connecting Tor...") + path = open('../../path.txt').readline().strip() + pro = subprocess.Popen(path) + pid = pro.pid + time.sleep(7.5) + input('Tor Connected. Press ENTER to continue\n') + return + + +# Login using premade account credentials and do login captcha manually +def login(driver): + #click login button + login_link = driver.find_element( + by=By.XPATH, value='/html/body/div/div[2]/div/table/tbody/tr[2]/td/center/pre/strong/a').\ + get_attribute('href') + driver.get(login_link) + + #entering username and password into input boxes + usernameBox = driver.find_element(by=By.XPATH, value='/html/body/div/div[2]/div/form/table/tbody/tr[2]/td[2]/input') + #Username here + usernameBox.send_keys('purely_cabbage') + passwordBox = driver.find_element(by=By.XPATH, value='/html/body/div/div[2]/div/form/table/tbody/tr[3]/td[2]/input') + #Password here + passwordBox.send_keys('$ourP@tchK1ds') + + input("Press ENTER when log in is completed\n") + + # wait for listing page show up (This Xpath may need to change based on different seed url) + WebDriverWait(driver, 50).until(EC.visibility_of_element_located( + (By.XPATH, '//*[@id="content"]'))) + + +# Returns the name of the website +def getForumName(): + name = 'OnniForums' + return name + + +# Return the link of the website +def getFixedURL(): + url = 'http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/' + return url + + +# Closes Tor Browser +def closetor(driver): + # global pid + # os.system("taskkill /pid " + str(pro.pid)) + # os.system("taskkill /t /f /im tor.exe") + print('Closing Tor...') + driver.close() + time.sleep(3) + return + + +# Creates FireFox 'driver' and configure its 'Profile' +# to use Tor proxy and socket +def createFFDriver(): + file = open('../../path.txt', 'r') + lines = file.readlines() + + ff_binary = FirefoxBinary(lines[0].strip()) + + ff_prof = FirefoxProfile(lines[1].strip()) + ff_prof.set_preference("places.history.enabled", False) + ff_prof.set_preference("privacy.clearOnShutdown.offlineApps", True) + ff_prof.set_preference("privacy.clearOnShutdown.passwords", True) + ff_prof.set_preference("privacy.clearOnShutdown.siteSettings", True) + ff_prof.set_preference("privacy.sanitize.sanitizeOnShutdown", True) + ff_prof.set_preference("signon.rememberSignons", False) + ff_prof.set_preference("network.cookie.lifetimePolicy", 2) + ff_prof.set_preference("network.dns.disablePrefetch", True)# + ff_prof.set_preference("network.http.sendRefererHeader", 0) + ff_prof.set_preference("permissions.default.image", 3) + ff_prof.set_preference("browser.download.folderList", 2) + ff_prof.set_preference("browser.download.manager.showWhenStarting", False) + ff_prof.set_preference("browser.helperApps.neverAsk.saveToDisk", "text/plain") + ff_prof.set_preference('network.proxy.type', 1) + ff_prof.set_preference("network.proxy.socks_version", 5) + ff_prof.set_preference('network.proxy.socks', '127.0.0.1') + ff_prof.set_preference('network.proxy.socks_port', 9150) + ff_prof.set_preference('network.proxy.socks_remote_dns', True) + ff_prof.set_preference("javascript.enabled", True) + ff_prof.update_preferences() + + service = Service(lines[2].strip()) + + driver = webdriver.Firefox(firefox_binary=ff_binary, firefox_profile=ff_prof, service=service) + + return driver + + +def getAccess(): + url = getFixedURL() + driver = createFFDriver() + try: + driver.get(url) + return driver + except: + driver.close() + return 'down' + + +# Saves the crawled html page +def savePage(page, url): + cleanPage = cleanHTML(page) + filePath = getFullPathName(url) + os.makedirs(os.path.dirname(filePath), exist_ok=True) + open(filePath, 'wb').write(cleanPage.encode('utf-8')) + return + + +# Gets the full path of the page to be saved along with its appropriate file name +def getFullPathName(url): + fileName = getNameFromURL(url) + if isDescriptionLink(url): + #..\CryptBB\HTML_Pages\\ + fullPath = r'..\OnniForums\HTML_Pages\\' + str( + "%02d" % date.today().month) + str("%02d" % date.today().day) + str( + "%04d" % date.today().year) + r'\\' + r'Description\\' + fileName + '.html' + else: + fullPath = r'..\OnniForums\HTML_Pages\\' + str( + "%02d" % date.today().month) + str("%02d" % date.today().day) + str( + "%04d" % date.today().year) + r'\\' + r'Listing\\' + fileName + '.html' + return fullPath + + +# Creates the file name from passed URL +def getNameFromURL(url): + global counter + name = ''.join(e for e in url if e.isalnum()) + if (name == ''): + name = str(counter) + counter = counter + 1 + return name + + +def getInterestedLinks(): + links = [] + + # Hacking & Cracking tutorials + links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Hacking-Cracking-tutorials') + # Hacking & Cracking questions + # links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Hacking-Cracking-questions') + # Exploit PoCs + # links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Exploit-PoCs') + # Cracked software + # links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Cracked-software') + # Malware-development + # links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Malware-development') + # Carding & Fraud + # links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Carding-Fraud') + # Darknet Discussions + # links.append('http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/forumdisplay.php?fid=88') + # OPSEC + # links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-OPSEC') + # Databases + # links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Databases') + # Proxies + # links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Proxies') + + return links + + +def crawlForum(driver): + print("Crawling the OnniForums forum") + + linksToCrawl = getInterestedLinks() + visited = set(linksToCrawl) + initialTime = time.time() + + i = 0 + count = 0 + while i < len(linksToCrawl): + link = linksToCrawl[i] + print('Crawling :', link) + try: + try: + driver.get(link) + except: + driver.refresh() + html = driver.page_source + savePage(html, link) + + has_next_page = True + while has_next_page: + list = topicPages(html) + for item in list: + itemURL = urlparse.urljoin(baseURL, str(item)) + try: + driver.get(itemURL) + except: + driver.refresh() + savePage(driver.page_source, item) + driver.back() + # comment out + break + + # comment out + #if count == 1: + #count = 0 + #break + + try: + temp = driver.find_element(by=By.XPATH, value= + '/html/body/div/div[2]/div/div[2]/div') + link = temp.find_element(by=By.CLASS_NAME, value='pagination_next').get_attribute('href') + + if link == "": + raise NoSuchElementException + try: + driver.get(link) + except: + driver.refresh() + html = driver.page_source + savePage(html, link) + count += 1 + + except NoSuchElementException: + has_next_page = False + + except Exception as e: + print(link, e) + i += 1 + + # finalTime = time.time() + # print finalTime - initialTime + + input("Crawling OnniForums forum done sucessfully. Press ENTER to continue\n") + + +# Returns 'True' if the link is Topic link +def isDescriptionLink(url): + if 'Thread' in url: + return True + return False + + +# Returns True if the link is a listingPage link +def isListingLink(url): + if 'Forum' in url: + return True + return False + + +# calling the parser to define the links +def topicPages(html): + soup = BeautifulSoup(html, "html.parser") + #print(soup.find('div', id="container").find('div', id="content").find('table', {"class": "tborder clear"}).find('tbody').find('tr',{"class": "inline_row"}).find('strong').text) + return cryptBB_links_parser(soup) + + +def crawler(): + startCrawling() + # print("Crawling and Parsing BestCardingWorld .... DONE!") diff --git a/Forums/CrackingPro/geckodriver.log b/Forums/CrackingPro/geckodriver.log new file mode 100644 index 0000000..d1e3763 --- /dev/null +++ b/Forums/CrackingPro/geckodriver.log @@ -0,0 +1,462 @@ +1655762162153 geckodriver INFO Listening on 127.0.0.1:50333 +1655762166434 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50334" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileqUeAAN" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655762166904 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50334/devtools/browser/fb880205-35da-44a4-83b8-a861ce7125f1 +1655762168590 Marionette INFO Listening on port 50341 +1655762169104 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655762207624 Marionette INFO Stopped listening on port 50341 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function + +###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + +1655762207923 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655762990320 geckodriver INFO Listening on 127.0.0.1:50884 +1655762994595 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50885" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileurjxEe" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655762995244 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50885/devtools/browser/36503f58-7ab7-4188-81b0-fa1b81f32c0a +1655762996997 Marionette INFO Listening on port 50890 +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +1655762997277 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655763083091 Marionette INFO Stopped listening on port 50890 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1655763083216 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655763089557 geckodriver INFO Listening on 127.0.0.1:50923 +1655763093759 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50924" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile5mY1qq" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655763094208 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50924/devtools/browser/68ce2df9-521f-4258-94f7-c2e2b199701a +1655763095918 Marionette INFO Listening on port 50929 +1655763096408 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655763143935 Marionette INFO Stopped listening on port 50929 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1655763144495 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655763166975 geckodriver INFO Listening on 127.0.0.1:50947 +1655763171175 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50948" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilenAGZM5" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655763171719 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50948/devtools/browser/0b9966bd-0cec-45cc-9d11-02a2580233f0 +1655763173307 Marionette INFO Listening on port 50953 +1655763173368 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655763208922 Marionette INFO Stopped listening on port 50953 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData] +!!! error running onStopped callback: TypeError: callback is not a function +1655763209261 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655763387409 geckodriver INFO Listening on 127.0.0.1:49752 +1655763391806 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "49753" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilevAHJSJ" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655763392827 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:49753/devtools/browser/7fe49a16-940f-4ee8-9a2f-48739a92db78 +1655763394700 Marionette INFO Listening on port 49759 +1655763395103 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655763430848 Marionette INFO Stopped listening on port 49759 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData] +!!! error running onStopped callback: TypeError: callback is not a function +1655763431145 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655765589286 geckodriver INFO Listening on 127.0.0.1:50102 +1655765621677 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50103" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile8ecUpb" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655765622149 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:50103/devtools/browser/e5a916fc-987c-43d4-9c12-9c9b88cca242 +1655765623776 Marionette INFO Listening on port 50109 +1655765623892 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655766125168 Marionette INFO Stopped listening on port 50109 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1655766125303 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655766505099 geckodriver INFO Listening on 127.0.0.1:50156 +1655766509351 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50157" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileVqvmzh" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655766509876 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50157/devtools/browser/13084c14-64d3-48a5-8b99-4c514e961d4f +1655766511328 Marionette INFO Listening on port 50162 +1655766511395 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655766552298 Marionette INFO Stopped listening on port 50162 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData] +!!! error running onStopped callback: TypeError: callback is not a function +1655766552578 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655767657126 geckodriver INFO Listening on 127.0.0.1:50257 +1655767661375 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50258" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileB9Dzeh" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655767661843 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50258/devtools/browser/867cd318-6430-4b3f-9b63-8c6bdf17636a +1655767663330 Marionette INFO Listening on port 50263 +1655767663431 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655767668263 Marionette INFO Stopped listening on port 50263 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function + +###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + +1655767668591 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655767689480 geckodriver INFO Listening on 127.0.0.1:50276 +1655767693635 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50277" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile8ZOQe7" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655767694215 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50277/devtools/browser/a70a3f5b-19af-4a68-99c8-c46086ba9599 +1655767695725 Marionette INFO Listening on port 50282 +1655767695830 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655767701272 Marionette INFO Stopped listening on port 50282 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1655767701594 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655767791360 geckodriver INFO Listening on 127.0.0.1:50296 +1655767795575 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50297" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile4GEqBr" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655767796001 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50297/devtools/browser/e2d4e57a-16ce-429d-9804-e9e391199ddf +1655767797534 Marionette INFO Listening on port 50302 +1655767797661 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655767802833 Marionette INFO Stopped listening on port 50302 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1655767803151 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655767902908 geckodriver INFO Listening on 127.0.0.1:50320 +1655767907009 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50321" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileqlJ8ZA" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655767907484 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:50321/devtools/browser/49e77669-01dd-447e-a804-36d42b3400cf +1655767908782 Marionette INFO Listening on port 50326 +1655767909069 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655767916218 Marionette INFO Stopped listening on port 50326 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1655767916523 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655767958348 geckodriver INFO Listening on 127.0.0.1:50340 +1655767962557 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50341" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilekpCdfs" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655767963032 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50341/devtools/browser/b4d1728c-d26f-4537-adc2-5c954b171b13 +1655767964591 Marionette INFO Listening on port 50346 +1655767964633 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655767973210 Marionette INFO Stopped listening on port 50346 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1655767973501 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655767998084 geckodriver INFO Listening on 127.0.0.1:50358 +1655768002314 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50359" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileujYNBj" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655768002757 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50359/devtools/browser/04112c45-204d-43f1-9615-34782fd06632 +1655768004165 Marionette INFO Listening on port 50364 +1655768004375 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655768009787 Marionette INFO Stopped listening on port 50364 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] + +###!!! [Child][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + +!!! error running onStopped callback: TypeError: callback is not a function + +###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + +1655768010100 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655768083966 geckodriver INFO Listening on 127.0.0.1:50386 +1655768088234 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50387" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile39Pl1f" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655768088664 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50387/devtools/browser/3d6cda42-cb60-4c67-8d76-1d186a51887f +1655768090263 Marionette INFO Listening on port 50392 +1655768090299 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655768091655 Marionette INFO Stopped listening on port 50392 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function + +###!!! [Parent][MessageChannel] Error: (msgtype=0x390076,name=PContent::Msg_DestroyBrowsingContextGroup) Closed channel: cannot send/recv + + +###!!! [Parent][MessageChannel] Error: (msgtype=0x390076,name=PContent::Msg_DestroyBrowsingContextGroup) Closed channel: cannot send/recv + + +###!!! [Parent][MessageChannel] Error: (msgtype=0x390076,name=PContent::Msg_DestroyBrowsingContextGroup) Closed channel: cannot send/recv + +1655768092659 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655768101201 geckodriver INFO Listening on 127.0.0.1:50404 +1655768105373 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50405" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile2YRGSz" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655768105853 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50405/devtools/browser/9c4a4d71-25ff-4980-a54d-8545d6200790 +1655768107431 Marionette INFO Listening on port 50410 +1655768107530 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655768119155 Marionette INFO Stopped listening on port 50410 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1655768119438 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655768229472 geckodriver INFO Listening on 127.0.0.1:50431 +1655768233458 webdriver::server WARN Rejected request with missing Host header diff --git a/Forums/CrackingPro/parser.py b/Forums/CrackingPro/parser.py new file mode 100644 index 0000000..902d4a1 --- /dev/null +++ b/Forums/CrackingPro/parser.py @@ -0,0 +1,354 @@ +__author__ = 'DarkWeb' + +# Here, we are importing the auxiliary functions to clean or convert data +from Forums.Utilities.utilities import * +from datetime import date +from datetime import timedelta +import re + +# Here, we are importing BeautifulSoup to search through the HTML tree +from bs4 import BeautifulSoup + +# This is the method to parse the Description Pages (one page to each topic in the Listing Pages) + +def cryptBB_description_parser(soup): + + # Fields to be parsed + + topic = "-1" # topic name + user = [] # all users of each post + addDate = [] # all dated of each post + feedback = [] # all feedbacks of each vendor (this was found in just one Forum and with a number format) + status = [] # all user's authority in each post such as (adm, member, dangerous) + reputation = [] # all users's karma in each post (usually found as a number) + sign = [] # all user's signature in each post (usually a standard message after the content of the post) + post = [] # all messages of each post + interest = [] # all user's interest in each post + + # Finding the topic (should be just one coming from the Listing Page) + + li = soup.find("td", {"class": "thead"}).find('strong') + topic = li.text + topic = re.sub("\[\w*\]", '', topic) + + topic = topic.replace(",","") + topic = topic.replace("\n","") + topic = cleanString(topic.strip()) + print(topic) + # Finding the repeated tag that corresponds to the listing of posts + + # posts = soup.find("form", {"name": "quickModForm"}).findAll('div', {"class": "windowbg"}) + \ + # soup.find("form", {"name": "quickModForm"}).findAll('div', {"class": "windowbg2"}) + + try: + posts = soup.find('table', {"class": "tborder tfixed clear"}).find('td', {"id": "posts_container"}).find_all( + 'div', {"class": "post"}) + # print(len(posts)) + + # For each message (post), get all the fields we are interested to: + + for ipost in posts: + + # Finding a first level of the HTML page + + # post_wrapper = ipost.find('div', {"class": "post_wrapper"}).find('div', {"class": "poster"}) + post_wrapper = ipost.find('span', {"class": "largetext"}) + # Finding the author (user) of the post + + # author = post_wrapper.find('h4') + author = post_wrapper.text.strip() + # print("author " + author) + user.append(cleanString(author)) # Remember to clean the problematic characters + + # Finding the status of the author + + smalltext = ipost.find('div', {"class": "post_author"}) + + # Testing here two possibilities to find this status and combine them + if ipost.find('div', {"class": "deleted_post_author"}): + status.append(-1) + interest.append(-1) + reputation.append(-1) + addDate.append(-1) + post.append("THIS POST HAS BEEN REMOVED!") + sign.append(-1) + feedback.append(-1) + continue + + # CryptBB does have membergroup and postgroup + + membergroup = smalltext.find('div', {"class": "profile-rank"}) + postgroup = smalltext.find('div', {"class": "postgroup"}) + if membergroup != None: + membergroup = membergroup.text.strip() + if postgroup != None: + postgroup = postgroup.text.strip() + membergroup = membergroup + " - " + postgroup + else: + if postgroup != None: + membergroup = postgroup.text.strip() + else: + membergroup = "-1" + + status.append(cleanString(membergroup)) + # print("status " + cleanString(membergroup)) + # Finding the interest of the author + # CryptBB does not have blurb + blurb = smalltext.find('li', {"class": "blurb"}) + if blurb != None: + blurb = blurb.text.strip() + else: + blurb = "-1" + interest.append(cleanString(blurb)) + + # Finding the reputation of the user + # CryptBB does have reputation + author_stats = smalltext.find('div', {"class": "author_statistics"}) + karma = author_stats.find('strong') + if karma != None: + karma = karma.text + karma = karma.replace("Community Rating: ", "") + karma = karma.replace("Karma: ", "") + karma = karma.strip() + else: + karma = "-1" + reputation.append(cleanString(karma)) + # print("karma " + cleanString(karma)) + # Getting here another good tag to find the post date, post content and users' signature + + postarea = ipost.find('div', {"class": "post_content"}) + + dt = postarea.find('span', {"class": "post_date"}).text + # dt = dt.strip().split() + dt = dt.strip() + day=date.today() + if "Yesterday" in dt: + yesterday = day - timedelta(days=1) + yesterday = yesterday.strftime('%m-%d-%Y') + stime = dt.replace('Yesterday,','').strip() + date_time_obj = yesterday+ ', '+stime + date_time_obj = datetime.strptime(date_time_obj,'%m-%d-%Y, %I:%M %p') + elif "hours ago" in dt: + day = day.strftime('%m-%d-%Y') + date_time_obj = postarea.find('span', {"class": "post_date"}).find('span')['title'] + date_time_obj = datetime.strptime(date_time_obj, '%m-%d-%Y, %I:%M %p') + else: + date_time_obj = datetime.strptime(dt, '%m-%d-%Y, %I:%M %p') + stime = date_time_obj.strftime('%b %d, %Y') + sdate = date_time_obj.strftime('%I:%M %p') + + + addDate.append(date_time_obj) + # print("date " + str(date_time_obj)) + # Finding the date of the post + # date_time_obj = datetime.strptime(dt, '%a %b %d, %Y %I:%M %p') + # smalltext = postarea.find('div', {"class": "flow_hidden"}).find('div', {"class": "keyinfo"})\ + # .find('div', {"class": "smalltext"}) + # sdatetime = smalltext.text + # sdatetime = sdatetime.replace(u"\xab","") # Removing unnecessary characters + # sdatetime = sdatetime.replace(u"\xbb","") # Removing unnecessary characters + # sdatetime = sdatetime.split("on: ") # Removing unnecessary characters + # sdatetime = sdatetime[1].strip() + # stime = sdatetime[:-12:-1] # Finding the time of the post + # stime = stime[::-1] + # sdate = sdatetime.replace(stime,"") # Finding the date of the post + # sdate = sdate.replace(",","") + # sdate = sdate.strip() + + # Covert the date of the post that can be informed as: "12 February 2016", "today", "yesterday". We need + # a date format here as "mm/dd/yyyy" + + # addDate.append(convertDate(sdate,"english", crawlerDate) + " " + stime) + + # Finding the post + + inner = postarea.find('div', {"class": "post_body scaleimages"}) + inner = inner.text.strip() + # print(inner) + post.append(cleanString(inner)) + + # Finding the users's signature + + # signature = ipost.find('div', {"class": "post_wrapper"}).find('div', {"class": "moderatorbar"}).find('div', {"class": "signature"}) + signature = ipost.find('div', {"class": "signature scaleimages"}) + if signature != None: + signature = signature.text.strip() + # print(signature) + else: + signature = "-1" + sign.append(cleanString(signature)) + + # As no information about users's feedback was found, just assign "-1" to the variable + + feedback.append("-1") + except: + if soup.find('td', {"class": "trow1"}).text == " You do not have permission to access this page. ": + user.append("-1") + status.append(-1) + interest.append(-1) + reputation.append(-1) + addDate.append(-1) + post.append("NO ACCESS TO THIS PAGE!") + sign.append(-1) + feedback.append(-1) + + + # Populate the final variable (this should be a list with all fields scraped) + + row = (topic, post, user, addDate, feedback, status, reputation, sign, interest) + + # Sending the results + + return row + +# This is the method to parse the Listing Pages (one page with many posts) + +def onniForums_listing_parser(soup): + + board = "-1" # board name (the previous level of the topic in the Forum categorization tree. + # For instance: Security/Malware/Tools to hack Facebook. The board here should be Malware) + + nm = 0 # this variable should receive the number of topics + topic = [] # all topics + user = [] # all users of each topic + post = [] # number of posts of each topic + view = [] # number of views of each topic + addDate = [] # when the topic was created (difficult to find) + href = [] # this variable should receive all cleaned urls (we will use this to do the marge between + # Listing and Description pages) + + # Finding the board (should be just one) + + board = soup.find('span', {"class": "active"}).text + board = cleanString(board.strip()) + + # Finding the repeated tag that corresponds to the listing of topics + + itopics = soup.find_all('tr',{"class": "inline_row"}) + index = 0 + for itopic in itopics: + + # For each topic found, the structure to get the rest of the information can be of two types. Testing all of them + # to don't miss any topic + + + # Adding the topic to the topic list + try: + topics = itopic.find('span', {"class": "subject_old"}).find('a').text + except: + topics = itopic.find('span', {"class": "subject_new"}).find('a').text + topics = re.sub("\[\w*\]", '', topics) + topic.append(cleanString(topics)) + + # Counting how many topics we have found so far + + nm = len(topic) + + # Adding the url to the list of urls + try: + link = itopic.find('span', {"class": "subject_old"}).find('a').get('href') + except: + link = itopic.find('span',{"class": "subject_new"}).find('a').get('href') + link = cleanLink(link) + href.append(link) + + # Finding the author of the topic + ps = itopic.find('div', {"class":"author smalltext"}).find('a').text + author = ps.strip() + user.append(cleanString(author)) + + # Finding the number of replies + columns = itopic.findChildren('td',recursive=False) + posts = columns[3].text + + post.append(cleanString(posts)) + + # Finding the number of Views + tview = columns[4].text + view.append(cleanString(tview)) + + + + # If no information about when the topic was added, just assign "-1" to the variable + #dt = itopic.find('div', {"class": "responsive-hide"}).text.split('ยป')[1] + #dt = dt.strip() + #date_time_obj = datetime.strptime(dt,'%a %b %d, %Y %I:%M %p') + #addDate.append(date_time_obj) + addDate.append("-1") + + + + index += 1 + return organizeTopics("CryptBB", nm, topic, board, view, post, user, addDate, href) + + # if len(tag) > 0: + # + # # Finding the topic + # + # tds = tds[0].find(tag[0]) + # topics = tds.text + # topics = topics.replace(u"\xbb","") + # topics = topics.strip() + # topic.append(cleanString(topics)) + # + # # Counting how many topics we have found so far + # + # nm = len(topic) + # + # # Adding the url to the list of urls + # + # link = tds.findAll('a', href=True) + # link = link[0].get('href') + # link = cleanLink(link) + # href.append(link) + # + # # Finding the author of the topic + # + # ps = itopic.find('td', {"class": tag[1]}).find('p').find('a') + # if ps == None: + # ps = itopic.find('td', {"class": tag[1]}).find('p') + # ps = ps.text.replace("Started by ","") + # else: + # ps = ps.text + # author = ps.strip() + # user.append(cleanString(author)) + # + # # Finding the number of replies + # + # statistics = itopic.find('td', {"class": tag[2]}) + # statistics = statistics.text + # statistics = statistics.split("Replies") + # posts = statistics[0].strip() + # post.append(cleanString(posts)) + # + # # Finding the number of Views + # + # views = statistics[1] + # views = views.replace("Views","") + # views = views.strip() + # view.append(cleanString(views)) + # + # # As no information about when the topic was added, just assign "-1" to the variable + # + # addDate.append("-1") + + #return organizeTopics("TheMajesticGarden", nm, topic, board, view, post, user, addDate, href) + +def cryptBB_links_parser(soup): + + # Returning all links that should be visited by the Crawler + + href = [] + #print(soup.find('table', {"class": "tborder clear"}).find( + # 'tbody').find_all('tr', {"class": "inline_row"})) + listing = soup.find('table', {"class": "tborder clear"}).find('tbody').find_all('tr', {"class": "inline_row"}) + + for a in listing: + try: + link = a.find('span', {"class": "subject_old"}).find('a').get('href') + except: + link = a.find('span', {"class": "subject_new"}).find('a').get('href') + + href.append(link) + + return href \ No newline at end of file diff --git a/Forums/CryptBB/crawler_mechanize.py b/Forums/CryptBB/crawler_mechanize.py index b37e3aa..99f3731 100644 --- a/Forums/CryptBB/crawler_mechanize.py +++ b/Forums/CryptBB/crawler_mechanize.py @@ -229,7 +229,6 @@ def findDescriptionPages(url): error = False try: - # C:\\Users\\CALSysLab\\Documents\\threatIntelligence-main\\DarkWebMining_Working\\Forums\\BestCardingWorld\\HTML_Pages\\ html = codecs.open( "C:\\Users\\CALSysLab\\Documents\\threatIntelligence-main\\DarkWebMining_Working\\Forums\\BestCardingWorld\\HTML_Pages\\" + str( "%02d" % date.today().month) + str("%02d" % date.today().day) + str( diff --git a/Forums/Initialization/geckodriver.log b/Forums/Initialization/geckodriver.log index 4bd12d3..d52daf0 100644 --- a/Forums/Initialization/geckodriver.log +++ b/Forums/Initialization/geckodriver.log @@ -2245,3 +2245,38 @@ unwatchForTargets()@TargetList.jsm:37 destructor()@TargetList.jsm:109 stop()@CDP.jsm:104 close()@RemoteAgent.jsm:138 +1686680656073 geckodriver INFO Listening on 127.0.0.1:52360 +1686680658779 mozrunner::runner INFO Running command: "C:\\Users\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "52361" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofileOia2w8" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: init()" +console.log: "TorConnect: Entering Initial state" +console.log: "TorConnect: Observed profile-after-change" +console.log: "TorConnect: Observing topic 'TorProcessExited'" +console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: Observing topic 'torsettings:ready'" +console.log: "TorSettings: Observed profile-after-change" +1686680659309 Marionette INFO Marionette enabled +console.log: "TorConnect: Will load after bootstrap => [about:blank]" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:52361/devtools/browser/cd72c598-1600-4715-a68f-1c0154e1158b +1686680660598 Marionette INFO Listening on port 52366 +1686680660733 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: https://onniforums.com/jscripts/thread.js?ver=1827, line 188: ReferenceError: spinner is not defined +JavaScript error: https://onniforums.com/jscripts/thread.js?ver=1827, line 188: ReferenceError: spinner is not defined +JavaScript error: https://onniforums.com/jscripts/thread.js?ver=1827, line 188: ReferenceError: spinner is not defined +1686680760568 Marionette INFO Stopped listening on port 52366 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first. +JavaScript error: resource://gre/modules/PageThumbs.jsm, line 709: AbortError: IOUtils.profileBeforeChange getter: IOUtils: profileBeforeChange phase has already finished +1686680761521 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 diff --git a/Forums/OnniForums/crawler_mechanize.py b/Forums/OnniForums/crawler_mechanize.py index 2580d9b..f243c41 100644 --- a/Forums/OnniForums/crawler_mechanize.py +++ b/Forums/OnniForums/crawler_mechanize.py @@ -67,7 +67,7 @@ def create_connection(address, timeout=None, source_address=None): # Returns the name of website def getForumName(): - name = 'BestCardingWorld' + name = 'OnniForums' return name @@ -110,12 +110,10 @@ def getAccess(): br = createBrowser() try: - br.open(url) return br except: - return 'down' @@ -132,11 +130,11 @@ def savePage(page, url): def getFullPathName(url): fileName = getNameFromURL(url) if isDescriptionLink(url): - fullPath = 'C:/Users/CALSysLab/Documents/threatIntelligence-main/DarkWebMining_Working/Forums/BestCardingWorld/HTML_Pages/' + str( + fullPath = 'C:/Users/CALSysLab/Documents/threatIntelligence-main/DarkWebMining_Working/Forums/OnniForums/HTML_Pages/' + str( "%02d" % date.today().month) + str("%02d" % date.today().day) + str( "%04d" % date.today().year) + '/' + 'Description/' + fileName + '.html' else: - fullPath = 'C:/Users/CALSysLab/Documents/threatIntelligence-main/DarkWebMining_Working/Forums/BestCardingWorld/HTML_Pages/' + str( + fullPath = 'C:/Users/CALSysLab/Documents/threatIntelligence-main/DarkWebMining_Working/Forums/OnniForums/HTML_Pages/' + str( "%02d" % date.today().month) + str("%02d" % date.today().day) + str( "%04d" % date.today().year) + '/' + 'Listing/' + fileName + '.html' return fullPath @@ -163,7 +161,7 @@ def getInterestedLinks(): # Start crawling Forum pages def crawlForum(br): - print("Crawling The Best Carding World forum") + print("Crawling OnniForum") linksToCrawl = getInterestedLinks() visited = set(linksToCrawl) @@ -230,14 +228,14 @@ def findDescriptionPages(url): error = False try: html = codecs.open( - "C:\\Users\\CALSysLab\\Documents\\threatIntelligence-main\\DarkWebMining_Working\\Forums\\BestCardingWorld\\HTML_Pages\\" + str( + "C:\\Users\\CALSysLab\\Documents\\threatIntelligence-main\\DarkWebMining_Working\\Forums\\OnniForums\\HTML_Pages\\" + str( "%02d" % date.today().month) + str("%02d" % date.today().day) + str( "%04d" % date.today().year) + "\\Listing\\" + getNameFromURL(url) + ".html", encoding='utf8') soup = BeautifulSoup(html, "html.parser") except: try: html = open( - "C:\\Users\\CALSysLab\\Documents\\threatIntelligence-main\\DarkWebMining_Working\\Forums\\BestCardingWorld\\HTML_Pages\\" + str( + "C:\\Users\\CALSysLab\\Documents\\threatIntelligence-main\\DarkWebMining_Working\\Forums\\OnniForums\\HTML_Pages\\" + str( "%02d" % date.today().month) + str("%02d" % date.today().day) + str( "%04d" % date.today().year) + "\\Listing\\" + getNameFromURL(url) + ".html") soup = BeautifulSoup(html, "html.parser") @@ -254,4 +252,4 @@ def findDescriptionPages(url): def crawler(): startCrawling() - print("Crawling and Parsing The Best Carding World .... DONE!") + print("Crawling and Parsing OnniForums .... DONE!")