started OnniForums crawler

1 year ago · ac97f4bece
--- a/.idea/DW_Pipeline_Test.iml
+++ b/.idea/DW_Pipeline_Test.iml
@ -2,7 +2,7 @@
 <module type="PYTHON_MODULE" version="4">
  <component name="NewModuleRootManager">
    <content url="file://$MODULE_DIR$" />
    <orderEntry type="inheritedJdk" />
    <orderEntry type="jdk" jdkName="C:\Users\Helium\anaconda3" jdkType="Python SDK" />
    <orderEntry type="sourceFolder" forTests="false" />
  </component>
  <component name="PyNamespacePackagesService">
--- a/.idea/misc.xml
+++ b/.idea/misc.xml
@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.9 (DW_Projects)" project-jdk-type="Python SDK" />
  <component name="ProjectRootManager" version="2" project-jdk-name="C:\Users\Helium\anaconda3" project-jdk-type="Python SDK" />
 </project>
--- a/Forums/BestCardingWorld/crawler_selenium.py
+++ b/Forums/BestCardingWorld/crawler_selenium.py
@ -96,7 +96,7 @@ def createFFDriver():
    ff_prof.set_preference("privacy.sanitize.sanitizeOnShutdown", True)
    ff_prof.set_preference("signon.rememberSignons", False)
    ff_prof.set_preference("network.cookie.lifetimePolicy", 2)
    ff_prof.set_preference("network.dns.disablePrefetch", True)
    ff_prof.set_preference("network.dns.disablePrefetch", True)#might need to turn off
    ff_prof.set_preference("network.http.sendRefererHeader", 0)
    ff_prof.set_preference("permissions.default.image", 2)
    ff_prof.set_preference("browser.download.folderList", 2)
--- a/Forums/CryptBB/crawler_selenium.py
+++ b/Forums/CryptBB/crawler_selenium.py
@ -36,7 +36,7 @@ def startCrawling():
    if driver != 'down':
        try:
            login(driver)
            crawlForum(driver)
            # crawlForum(driver)
        except Exception as e:
            print(driver.current_url, e)
        closetor(driver)
@ -144,7 +144,7 @@ def createFFDriver():
    ff_prof.set_preference("privacy.sanitize.sanitizeOnShutdown", True)
    ff_prof.set_preference("signon.rememberSignons", False)
    ff_prof.set_preference("network.cookie.lifetimePolicy", 2)
    ff_prof.set_preference("network.dns.disablePrefetch", True)
    ff_prof.set_preference("network.dns.disablePrefetch", True)#
    ff_prof.set_preference("network.http.sendRefererHeader", 0)
    ff_prof.set_preference("permissions.default.image", 3)
    ff_prof.set_preference("browser.download.folderList", 2)
--- a/Forums/Initialization/forumsList.txt
+++ b/Forums/Initialization/forumsList.txt
@ -1 +1 @@
 CryptBB
 CryptBB
--- a/Forums/Initialization/forums_mining.py
+++ b/Forums/Initialization/forums_mining.py
@ -8,11 +8,12 @@ import os
 from datetime import *
 from Forums.BestCardingWorld.crawler_selenium import crawler as crawlerBestCardingWorld
 from Forums.CryptBB.crawler_selenium import crawler as crawlerCryptBB
 from Forums.OnniForums.crawler_selenium import crawler as crawlerOnniForums
 import time
 # reads list of marketplaces
 # reads list of marketplaces manually inputted
 def getForums():
    forums = []
    with open('forumsList.txt') as f:
@ -77,22 +78,27 @@ def createSubdirectories(pagesDir):
        os.mkdir(descReadDir)
 #main method
 if __name__ == '__main__':
    print("HelloWorld")
    #assignment from forumsList.txt
    forumsList = getForums()
    #get forum from forumsList
    for forum in forumsList:
        forum = forum.replace('\n','')
        print("Creating listing and description directories ...")
        createDirectory(forum)
        time.sleep(5)
        time.sleep(5) #wait for directories to be created
        input("Directories created successfully. Press ENTER to continue\n")
        if forum == "BestCardingWorld":
            crawlerBestCardingWorld()
        elif forum == "CryptBB":
            crawlerCryptBB()
        elif forum == "OnniForums":
            crawlerOnniForums()
    print("Scraping process completed successfully!")
--- a/Forums/Initialization/geckodriver.log
+++ b/Forums/Initialization/geckodriver.log
@ -1402,3 +1402,310 @@ unwatchForTargets()@TargetList.jsm:37
 destructor()@TargetList.jsm:109
 stop()@CDP.jsm:104
 close()@RemoteAgent.jsm:138
 1686338884789	geckodriver	INFO	Listening on 127.0.0.1:51464
 1686338889554	mozrunner::runner	INFO	Running command: "C:\\Users\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "51465" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofilePx8hdP"
 console.log: "TorSettings: loadFromPrefs()"
 console.log: "TorConnect: init()"
 console.log: "TorConnect: Entering Initial state"
 console.log: "TorConnect: Observed profile-after-change"
 console.log: "TorConnect: Observing topic 'TorProcessExited'"
 console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'"
 console.log: "TorConnect: Observing topic 'torsettings:ready'"
 console.log: "TorSettings: Observed profile-after-change"
 1686338895681	Marionette	INFO	Marionette enabled
 console.log: "TorConnect: Will load after bootstrap => [about:blank]"
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
 DevTools listening on ws://localhost:51465/devtools/browser/ca518483-f04e-427e-89d6-a996307996a5
 1686338902642	Marionette	INFO	Listening on port 51470
 1686338902983	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
 JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/, line 2: ReferenceError: $ is not defined
 JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 2: ReferenceError: $ is not defined
 JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 5: ReferenceError: lang is not defined
 JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 9: ReferenceError: use_xmlhttprequest is not defined
 1686338987650	Marionette	INFO	Stopped listening on port 51470
 JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
 [Parent 5940, IPC I/O Parent] WARNING: file /var/tmp/build/firefox-b6010b1466c9/ipc/chromium/src/base/process_util_win.cc:167
 !!! error running onStopped callback: TypeError: callback is not a function
 ###!!! [Child][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost
 JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first.
 JavaScript error: resource://gre/modules/PageThumbs.jsm, line 709: AbortError: IOUtils.profileBeforeChange getter: IOUtils: profileBeforeChange phase has already finished
 ###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost
 1686338988537	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
 unwatchForTabs()@TargetList.jsm:70
 unwatchForTargets()@TargetList.jsm:37
 destructor()@TargetList.jsm:109
 stop()@CDP.jsm:104
 close()@RemoteAgent.jsm:138
 1686596562082	geckodriver	INFO	Listening on 127.0.0.1:50187
 1686596566629	mozrunner::runner	INFO	Running command: "C:\\Users\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50188" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofileQhGGOx"
 console.log: "TorSettings: loadFromPrefs()"
 console.log: "TorConnect: init()"
 console.log: "TorConnect: Entering Initial state"
 console.log: "TorConnect: Observed profile-after-change"
 console.log: "TorConnect: Observing topic 'TorProcessExited'"
 console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'"
 console.log: "TorConnect: Observing topic 'torsettings:ready'"
 console.log: "TorSettings: Observed profile-after-change"
 1686596576118	Marionette	INFO	Marionette enabled
 console.log: "TorConnect: Will load after bootstrap => [about:blank]"
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
 JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object.
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 DevTools listening on ws://localhost:50188/devtools/browser/3324b32e-574b-4b12-8ad5-138d84a7b3b0
 1686596590433	Marionette	INFO	Listening on port 50204
 1686596590541	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
 JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/, line 2: ReferenceError: $ is not defined
 JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 2: ReferenceError: $ is not defined
 JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 5: ReferenceError: lang is not defined
 JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 9: ReferenceError: use_xmlhttprequest is not defined
 1686596652376	Marionette	INFO	Stopped listening on port 50204
 JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
 !!! error running onStopped callback: TypeError: callback is not a function
 JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first.
 JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofileQhGGOx\thumbnails) because it does not exist
 1686596653163	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
 unwatchForTabs()@TargetList.jsm:70
 unwatchForTargets()@TargetList.jsm:37
 destructor()@TargetList.jsm:109
 stop()@CDP.jsm:104
 close()@RemoteAgent.jsm:138
 1686605562853	geckodriver	INFO	Listening on 127.0.0.1:50934
 1686605574898	mozrunner::runner	INFO	Running command: "C:\\Users\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50935" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofileDnVqxJ"
 console.log: "TorSettings: loadFromPrefs()"
 console.log: "TorConnect: init()"
 console.log: "TorConnect: Entering Initial state"
 console.log: "TorConnect: Observed profile-after-change"
 console.log: "TorConnect: Observing topic 'TorProcessExited'"
 console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'"
 console.log: "TorConnect: Observing topic 'torsettings:ready'"
 console.log: "TorSettings: Observed profile-after-change"
 1686605580062	Marionette	INFO	Marionette enabled
 console.log: "TorConnect: Will load after bootstrap => [about:blank]"
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
 DevTools listening on ws://localhost:50935/devtools/browser/31caaf51-c751-4c17-b821-bbe5b1b5eb18
 1686605587267	Marionette	INFO	Listening on port 50964
 1686605587958	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
 [Parent 8804, IPC I/O Parent] WARNING: file /var/tmp/build/firefox-b6010b1466c9/ipc/chromium/src/base/process_util_win.cc:167
 JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/, line 2: ReferenceError: $ is not defined
 JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 2: ReferenceError: $ is not defined
 JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 5: ReferenceError: lang is not defined
 JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 9: ReferenceError: use_xmlhttprequest is not defined
 console.log: "TorConnect: beginBootstrap()"
 console.log: "TorConnect: Try transitioning from Initial to Bootstrapping"
 console.log: "TorConnect: Exited Initial state"
 console.log: "TorConnect: Entering Bootstrapping state"
 JavaScript error: resource://gre/actors/RemotePageChild.jsm, line 116: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIXPCComponents_Utils.cloneInto]
 1686605733273	Marionette	INFO	Stopped listening on port 50964
 JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
 JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData]
 !!! error running onStopped callback: TypeError: callback is not a function
 JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first.
 JavaScript error: resource://gre/modules/PageThumbs.jsm, line 709: AbortError: IOUtils.profileBeforeChange getter: IOUtils: profileBeforeChange phase has already finished
 [Parent 8804, IPC I/O Parent] WARNING: pipe error: 232: file /var/tmp/build/firefox-b6010b1466c9/ipc/chromium/src/chrome/common/ipc_channel_win.cc:544
 1686605733655	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
 unwatchForTabs()@TargetList.jsm:70
 unwatchForTargets()@TargetList.jsm:37
 destructor()@TargetList.jsm:109
 stop()@CDP.jsm:104
 close()@RemoteAgent.jsm:138
 1686609198317	geckodriver	INFO	Listening on 127.0.0.1:51716
 1686609202661	mozrunner::runner	INFO	Running command: "C:\\Users\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "51717" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofilei8xuPT"
 console.log: "TorSettings: loadFromPrefs()"
 console.log: "TorConnect: init()"
 console.log: "TorConnect: Entering Initial state"
 console.log: "TorConnect: Observed profile-after-change"
 console.log: "TorConnect: Observing topic 'TorProcessExited'"
 console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'"
 console.log: "TorConnect: Observing topic 'torsettings:ready'"
 console.log: "TorSettings: Observed profile-after-change"
 1686609203765	Marionette	INFO	Marionette enabled
 console.log: "TorConnect: Will load after bootstrap => [about:blank]"
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
 DevTools listening on ws://localhost:51717/devtools/browser/9102313b-d646-4ee2-a043-875c18e99d72
 1686609206596	Marionette	INFO	Listening on port 51725
 1686609207200	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
 JavaScript error: http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/, line 3: ReferenceError: lang is not defined
 JavaScript error: http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/, line 34: ReferenceError: $ is not defined
 JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofilei8xuPT\thumbnails) because it does not exist
 1686609237384	Marionette	INFO	Stopped listening on port 51725
 JavaScript error: resource://gre/modules/AsyncShutdown.jsm, line 575: NotFoundError: No such JSWindowActor 'MarionetteEvents'
 JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
 !!! error running onStopped callback: TypeError: callback is not a function
 JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first.
 JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofilei8xuPT\thumbnails) because it does not exist
 1686609238028	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
 unwatchForTabs()@TargetList.jsm:70
 unwatchForTargets()@TargetList.jsm:37
 destructor()@TargetList.jsm:109
 stop()@CDP.jsm:104
 close()@RemoteAgent.jsm:138
 1686609367764	geckodriver	INFO	Listening on 127.0.0.1:51806
 1686609371554	mozrunner::runner	INFO	Running command: "C:\\Users\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "51807" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofilesBnBWf"
 console.log: "TorSettings: loadFromPrefs()"
 console.log: "TorConnect: init()"
 console.log: "TorConnect: Entering Initial state"
 console.log: "TorConnect: Observed profile-after-change"
 console.log: "TorConnect: Observing topic 'TorProcessExited'"
 console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'"
 console.log: "TorConnect: Observing topic 'torsettings:ready'"
 console.log: "TorSettings: Observed profile-after-change"
 1686609372356	Marionette	INFO	Marionette enabled
 console.log: "TorConnect: Will load after bootstrap => [about:blank]"
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
 DevTools listening on ws://localhost:51807/devtools/browser/f7991b13-8912-433b-8eea-642199ceedab
 1686609375774	Marionette	INFO	Listening on port 51812
 1686609375994	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
 JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofilesBnBWf\thumbnails) because it does not exist
 1686609438764	Marionette	INFO	Stopped listening on port 51812
 JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
 !!! error running onStopped callback: TypeError: callback is not a function
 JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first.
 JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofilesBnBWf\thumbnails) because it does not exist
 1686609439513	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
 unwatchForTabs()@TargetList.jsm:70
 unwatchForTargets()@TargetList.jsm:37
 destructor()@TargetList.jsm:109
 stop()@CDP.jsm:104
 close()@RemoteAgent.jsm:138
 1686609756680	geckodriver	INFO	Listening on 127.0.0.1:51911
 1686609760338	mozrunner::runner	INFO	Running command: "C:\\Users\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "51912" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofile9AutkK"
 console.log: "TorSettings: loadFromPrefs()"
 console.log: "TorConnect: init()"
 console.log: "TorConnect: Entering Initial state"
 console.log: "TorConnect: Observed profile-after-change"
 console.log: "TorConnect: Observing topic 'TorProcessExited'"
 console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'"
 console.log: "TorConnect: Observing topic 'torsettings:ready'"
 console.log: "TorSettings: Observed profile-after-change"
 1686609761075	Marionette	INFO	Marionette enabled
 console.log: "TorConnect: Will load after bootstrap => [about:blank]"
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
 DevTools listening on ws://localhost:51912/devtools/browser/72ad40a2-a5be-4660-b604-94a3f50327fb
 1686609763168	Marionette	INFO	Listening on port 51917
 1686609763658	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
 JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/, line 2: ReferenceError: $ is not defined
 JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 2: ReferenceError: $ is not defined
 JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 5: ReferenceError: lang is not defined
 JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 9: ReferenceError: use_xmlhttprequest is not defined
 console.log: "TorConnect: beginBootstrap()"
 console.log: "TorConnect: Try transitioning from Initial to Bootstrapping"
 console.log: "TorConnect: Exited Initial state"
 JavaScript error: resource:///modules/TorConnectParent.jsm, line 110: InvalidStateError: JSWindowActorParent.sendAsyncMessage: JSWindowActorParent cannot send at the moment
 console.log: "TorConnect: Entering Bootstrapping state"
 JavaScript error: resource://gre/actors/RemotePageChild.jsm, line 116: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIXPCComponents_Utils.cloneInto]
 1686609860153	Marionette	INFO	Stopped listening on port 51917
 JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
 !!! error running onStopped callback: TypeError: callback is not a function
 JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first.
 JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofile9AutkK\thumbnails) because it does not exist
 [Parent 2244, IPC I/O Parent] WARNING: file /var/tmp/build/firefox-b6010b1466c9/ipc/chromium/src/base/process_util_win.cc:167
 1686609860787	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
 unwatchForTabs()@TargetList.jsm:70
 unwatchForTargets()@TargetList.jsm:37
 destructor()@TargetList.jsm:109
 stop()@CDP.jsm:104
 close()@RemoteAgent.jsm:138
 1686610746918	geckodriver	INFO	Listening on 127.0.0.1:52097
 1686610750204	mozrunner::runner	INFO	Running command: "C:\\Users\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "52098" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofileMks8y4"
 console.log: "TorSettings: loadFromPrefs()"
 console.log: "TorConnect: init()"
 console.log: "TorConnect: Entering Initial state"
 console.log: "TorConnect: Observed profile-after-change"
 console.log: "TorConnect: Observing topic 'TorProcessExited'"
 console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'"
 console.log: "TorConnect: Observing topic 'torsettings:ready'"
 console.log: "TorSettings: Observed profile-after-change"
 1686610751007	Marionette	INFO	Marionette enabled
 console.log: "TorConnect: Will load after bootstrap => [about:blank]"
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
 DevTools listening on ws://localhost:52098/devtools/browser/57fa325f-b035-44b6-a179-7f6407f259a5
 1686610753158	Marionette	INFO	Listening on port 52103
 1686610753378	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
 1686610846274	Marionette	INFO	Stopped listening on port 52103
 JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
 !!! error running onStopped callback: TypeError: callback is not a function
 JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first.
 JavaScript error: resource://gre/modules/PageThumbs.jsm, line 709: AbortError: IOUtils.profileBeforeChange getter: IOUtils: profileBeforeChange phase has already finished
 1686610846887	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
 unwatchForTabs()@TargetList.jsm:70
 unwatchForTargets()@TargetList.jsm:37
 destructor()@TargetList.jsm:109
 stop()@CDP.jsm:104
 close()@RemoteAgent.jsm:138
 1686611138064	geckodriver	INFO	Listening on 127.0.0.1:52201
 1686611140362	mozrunner::runner	INFO	Running command: "C:\\Users\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "52202" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofileCzAsf4"
 console.log: "TorSettings: loadFromPrefs()"
 console.log: "TorConnect: init()"
 console.log: "TorConnect: Entering Initial state"
 console.log: "TorConnect: Observed profile-after-change"
 console.log: "TorConnect: Observing topic 'TorProcessExited'"
 console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'"
 console.log: "TorConnect: Observing topic 'torsettings:ready'"
 console.log: "TorSettings: Observed profile-after-change"
 1686611140911	Marionette	INFO	Marionette enabled
 console.log: "TorConnect: Will load after bootstrap => [about:blank]"
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
 DevTools listening on ws://localhost:52202/devtools/browser/fda7fcbe-469d-43ad-a4dd-5eebc6b30bc6
 1686611142583	Marionette	INFO	Listening on port 52207
 1686611142902	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
 JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/, line 2: ReferenceError: $ is not defined
 JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 2: ReferenceError: $ is not defined
 JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 5: ReferenceError: lang is not defined
 JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 9: ReferenceError: use_xmlhttprequest is not defined
 JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 2: ReferenceError: $ is not defined
 JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 5: ReferenceError: lang is not defined
 JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 9: ReferenceError: use_xmlhttprequest is not defined
 1686611206885	Marionette	INFO	Stopped listening on port 52207
 JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
 !!! error running onStopped callback: TypeError: callback is not a function
 JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first.
 JavaScript error: resource://gre/modules/PageThumbs.jsm, line 709: AbortError: IOUtils.profileBeforeChange getter: IOUtils: profileBeforeChange phase has already finished
 1686611207399	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
 unwatchForTabs()@TargetList.jsm:70
 unwatchForTargets()@TargetList.jsm:37
 destructor()@TargetList.jsm:109
 stop()@CDP.jsm:104
 close()@RemoteAgent.jsm:138
--- a/Forums/OnniForums/captcha.png
+++ b/Forums/OnniForums/captcha.png
--- a/Forums/OnniForums/crawler_mechanize.py
+++ b/Forums/OnniForums/crawler_mechanize.py
@ -0,0 +1,257 @@
 __author__ = '91Shadows'
 '''
 BestCardingWorld Crawler (Mechanize)
 '''
 import codecs, os, re
 import socks, socket, time
 from datetime import date
 import urllib.parse as urlparse
 import http.client as httplib
 import mechanize
 import subprocess
 from bs4 import BeautifulSoup
 from Forums.Initialization.prepare_parser import new_parse
 from Forums.BestCardingWorld.parser import bestcardingworld_links_parser
 counter = 1
 httplib.HTTPConnection._http_vsn = 10
 httplib.HTTPConnection._http_vsn_str = 'HTTP/1.0'
 baseURL = 'http://bestteermb42clir6ux7xm76d4jjodh3fpahjqgbddbmfrgp4skg2wqd.onion/viewforum.php?f=42&sid=ee2cbfd73c12923d979790b2bb4bdfd5'
 socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, "127.0.0.1", 9150)
 # Opens Tor Browser, crawls the website
 def startCrawling():
    opentor()
    getUrl()
    forumName = getForumName()
    br = getAccess()
    if br != 'down':
        crawlForum(br)
        new_parse(forumName, False)
    # new_parse(forumName, False)
    closetor()
 # Opens Tor Browser
 def opentor():
    global pid
    print("Connecting Tor...")
    path = open('../../path.txt').readline()
    pro = subprocess.Popen(path)
    pid = pro.pid
    time.sleep(7.5)
    input("Tor Connected. Press ENTER to continue\n")
    return
 # Creates a connection through Tor Port
 def getUrl(timeout=None):
    socket.socket = socks.socksocket
    socket.create_connection = create_connection
    return
 # Makes the onion address request
 def create_connection(address, timeout=None, source_address=None):
    sock = socks.socksocket()
    sock.connect(address)
    return sock
 # Returns the name of website
 def getForumName():
    name = 'BestCardingWorld'
    return name
 # Return the link of website
 def getFixedURL():
    url = 'http://bestteermb42clir6ux7xm76d4jjodh3fpahjqgbddbmfrgp4skg2wqd.onion/viewforum.php?f=42&sid=ee2cbfd73c12923d979790b2bb4bdfd5'
    return url
 # Closes Tor Browser
 def closetor():
    global pid
    os.system("taskkill /pid " + str(pid))
    print('Closing Tor...')
    time.sleep(3)
    return
 # Creates a Mechanize browser and initializes its options
 def createBrowser():
    br = mechanize.Browser()
    cj = mechanize.CookieJar()
    br.set_cookiejar(cj)
    # Browser options
    br.set_handle_equiv(True)
    br.set_handle_redirect(True)
    br.set_handle_referer(True)
    br.set_handle_robots(False)
    br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
    br.addheaders = [('User-agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko'),
                     ('Accept', '*/*')]
    return br
 def getAccess():
    url = getFixedURL()
    br = createBrowser()
    try:
        br.open(url)
        return br
    except:
        return 'down'
 # Saves the crawled html page
 def savePage(page, url):
    filePath = getFullPathName(url)
    os.makedirs(os.path.dirname(filePath), exist_ok=True)
    a = page.read()
    open(filePath, "wb").write(a)
    return
 # Gets the full path of the page to be saved along with its appropriate file name
 def getFullPathName(url):
    fileName = getNameFromURL(url)
    if isDescriptionLink(url):
        fullPath = 'C:/Users/CALSysLab/Documents/threatIntelligence-main/DarkWebMining_Working/Forums/BestCardingWorld/HTML_Pages/' + str(
            "%02d" % date.today().month) + str("%02d" % date.today().day) + str(
            "%04d" % date.today().year) + '/' + 'Description/' + fileName + '.html'
    else:
        fullPath = 'C:/Users/CALSysLab/Documents/threatIntelligence-main/DarkWebMining_Working/Forums/BestCardingWorld/HTML_Pages/' + str(
            "%02d" % date.today().month) + str("%02d" % date.today().day) + str(
            "%04d" % date.today().year) + '/' + 'Listing/' + fileName + '.html'
    return fullPath
 # Creates the name of the file based on URL
 def getNameFromURL(url):
    global counter
    name = ''.join(e for e in url if e.isalnum())
    if (name == ''):
        name = str(counter)
        counter = counter + 1
    return name
 # Hacking and Markets related topics
 def getInterestedLinks():
    links = []
    links.append('http://bestteermb42clir6ux7xm76d4jjodh3fpahjqgbddbmfrgp4skg2wqd.onion/viewforum.php?f=43&sid=e12864ffccc5df877b03b573534955be')
    return links
 # Start crawling Forum pages
 def crawlForum(br):
    print("Crawling The Best Carding World forum")
    linksToCrawl = getInterestedLinks()
    visited = set(linksToCrawl)
    initialTime = time.time()
    i = 0
    while i < len(linksToCrawl):
        link = linksToCrawl[i]
        print('Crawling :', link)
        try:
            page = br.open(link)
            savePage(page, link)
            res = br.response().read()
            soup = BeautifulSoup(res, 'html.parser')
            next_link = soup.find("a", {"rel": "next"})
            if next_link != None:
                full_url = urlparse.urljoin(linksToCrawl[i], next_link['href'])
                linksToCrawl.insert(i + 1, full_url)
            listOfTopics = findDescriptionPages(link)
            for topic in listOfTopics:
                itemPage = br.open(str(topic))
                savePage(itemPage, topic)
        except Exception as e:
            print('Error getting link: ', link, e)
        i += 1
    # finalTime = time.time()
    # print finalTime - initialTime
    input("Crawling Best Carding world forum done sucessfully. Press ENTER to continue\n")
    return
 # Returns True if the link is 'Topic' Links
 def isDescriptionLink(url):
    if 'topic' in url:
        return True
    return False
 # Returns True if the link is a listingPage link
 def isListingLink(url):
    '''
    reg = 'board=[0-9]+.[0-9]+\Z'
    if len(re.findall(reg, url)) == 0:
        return False
    return True
    '''
    if 'forum' in url:
        return True
    return False
 # calling the parser to define the links
 def findDescriptionPages(url):
    soup = ""
    error = False
    try:
        html = codecs.open(
            "C:\\Users\\Helium\\Documents\\threatIntelligence-main\\DarkWebMining_Working\\Forums\\OnniForums\\HTML_Pages\\" + str(
                "%02d" % date.today().month) + str("%02d" % date.today().day) + str(
                "%04d" % date.today().year) + "\\Listing\\" + getNameFromURL(url) + ".html", encoding='utf8')
        soup = BeautifulSoup(html, "html.parser")
    except:
        try:
            html = open(
                "C:\\Users\\Helium\\Documents\\threatIntelligence-main\\DarkWebMining_Working\\Forums\\OnniForums\\HTML_Pages\\" + str(
                    "%02d" % date.today().month) + str("%02d" % date.today().day) + str(
                    "%04d" % date.today().year) + "\\Listing\\" + getNameFromURL(url) + ".html")
            soup = BeautifulSoup(html, "html.parser")
        except:
            error = True
            print("There was a problem to read the file " + getNameFromURL(url) + " in the listing section.")
    if not error:
        return bestcardingworld_links_parser(soup)
    else:
        return []
 def crawler():
    startCrawling()
    print("Crawling and Parsing The Best Carding World .... DONE!")
--- a/Forums/OnniForums/crawler_selenium.py
+++ b/Forums/OnniForums/crawler_selenium.py
@ -0,0 +1,303 @@
 __author__ = 'DarkWeb'
 '''
 OnniForums Crawler (Selenium)
 '''
 from selenium import webdriver
 from selenium.common.exceptions import NoSuchElementException
 from selenium.webdriver.firefox.firefox_profile import FirefoxProfile
 from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
 from selenium.webdriver.firefox.service import Service
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import WebDriverWait
 from PIL import Image
 import urllib.parse as urlparse
 import os, re, time
 from datetime import date
 import subprocess
 from bs4 import BeautifulSoup
 from Forums.Initialization.prepare_parser import new_parse
 from Forums.OnniForums.parser import cryptBB_links_parser
 from Forums.Utilities.utilities import cleanHTML
 counter = 1
 baseURL = 'http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/'
 # Opens Tor Browser, crawls the website
 def startCrawling():
    opentor()
    # forumName = getForumName()
    driver = getAccess()
    if driver != 'down':
        try:
            login(driver)
            # crawlForum(driver)
        except Exception as e:
            print(driver.current_url, e)
        closetor(driver)
    # new_parse(forumName, False)
 # Opens Tor Browser
 def opentor():
    global pid
    print("Connecting Tor...")
    path = open('../../path.txt').readline().strip()
    pro = subprocess.Popen(path)
    pid = pro.pid
    time.sleep(7.5)
    input('Tor Connected. Press ENTER to continue\n')
    return
 # Login using premade account credentials and do login captcha manually
 def login(driver):
    #click login button
    login_link = driver.find_element(
                by=By.XPATH, value='/html/body/div/div[2]/div/table/tbody/tr[2]/td/center/pre/strong/a').\
                get_attribute('href')
    driver.get(login_link)
    #entering username and password into input boxes
    usernameBox = driver.find_element(by=By.XPATH, value='/html/body/div/div[2]/div/form/table/tbody/tr[2]/td[2]/input')
    #Username here
    usernameBox.send_keys('purely_cabbage')
    passwordBox = driver.find_element(by=By.XPATH, value='/html/body/div/div[2]/div/form/table/tbody/tr[3]/td[2]/input')
    #Password here
    passwordBox.send_keys('$ourP@tchK1ds')
    input("Press ENTER when log in is completed\n")
    # wait for listing page show up (This Xpath may need to change based on different seed url)
    WebDriverWait(driver, 50).until(EC.visibility_of_element_located(
        (By.XPATH, '//*[@id="tab_content"]')))
 # Returns the name of the website
 def getForumName():
    name = 'OnniForums'
    return name
 # Return the link of the website
 def getFixedURL():
    url = 'http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/'
    return url
 # Closes Tor Browser
 def closetor(driver):
    # global pid
    # os.system("taskkill /pid " + str(pro.pid))
    # os.system("taskkill /t /f /im tor.exe")
    print('Closing Tor...')
    driver.close()
    time.sleep(3)
    return
 # Creates FireFox 'driver' and configure its 'Profile'
 # to use Tor proxy and socket
 def createFFDriver():
    file = open('../../path.txt', 'r')
    lines = file.readlines()
    ff_binary = FirefoxBinary(lines[0].strip())
    ff_prof = FirefoxProfile(lines[1].strip())
    ff_prof.set_preference("places.history.enabled", False)
    ff_prof.set_preference("privacy.clearOnShutdown.offlineApps", True)
    ff_prof.set_preference("privacy.clearOnShutdown.passwords", True)
    ff_prof.set_preference("privacy.clearOnShutdown.siteSettings", True)
    ff_prof.set_preference("privacy.sanitize.sanitizeOnShutdown", True)
    ff_prof.set_preference("signon.rememberSignons", False)
    ff_prof.set_preference("network.cookie.lifetimePolicy", 2)
    ff_prof.set_preference("network.dns.disablePrefetch", True)#
    ff_prof.set_preference("network.http.sendRefererHeader", 0)
    ff_prof.set_preference("permissions.default.image", 3)
    ff_prof.set_preference("browser.download.folderList", 2)
    ff_prof.set_preference("browser.download.manager.showWhenStarting", False)
    ff_prof.set_preference("browser.helperApps.neverAsk.saveToDisk", "text/plain")
    ff_prof.set_preference('network.proxy.type', 1)
    ff_prof.set_preference("network.proxy.socks_version", 5)
    ff_prof.set_preference('network.proxy.socks', '127.0.0.1')
    ff_prof.set_preference('network.proxy.socks_port', 9150)
    ff_prof.set_preference('network.proxy.socks_remote_dns', True)
    ff_prof.set_preference("javascript.enabled", True)
    ff_prof.update_preferences()
    service = Service(lines[2].strip())
    driver = webdriver.Firefox(firefox_binary=ff_binary, firefox_profile=ff_prof, service=service)
    return driver
 def getAccess():
    url = getFixedURL()
    driver = createFFDriver()
    try:
        driver.get(url)
        return driver
    except:
        driver.close()
        return 'down'
 # Saves the crawled html page
 def savePage(page, url):
    cleanPage = cleanHTML(page)
    filePath = getFullPathName(url)
    os.makedirs(os.path.dirname(filePath), exist_ok=True)
    open(filePath, 'wb').write(cleanPage.encode('utf-8'))
    return
 # Gets the full path of the page to be saved along with its appropriate file name
 def getFullPathName(url):
    fileName = getNameFromURL(url)
    if isDescriptionLink(url):
        fullPath = r'..\OnniForums\HTML_Pages\\' + str(
            "%02d" % date.today().month) + str("%02d" % date.today().day) + str(
            "%04d" % date.today().year) + r'\\' + r'Description\\' + fileName + '.html'
    else:
        fullPath = r'..\OnniForums\HTML_Pages\\' + str(
            "%02d" % date.today().month) + str("%02d" % date.today().day) + str(
            "%04d" % date.today().year) + r'\\' + r'Listing\\' + fileName + '.html'
    return fullPath
 # Creates the file name from passed URL
 def getNameFromURL(url):
    global counter
    name = ''.join(e for e in url if e.isalnum())
    if (name == ''):
        name = str(counter)
        counter = counter + 1
    return name
 def getInterestedLinks():
    links = []
    # Hacking & Cracking tutorials
    links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Hacking-Cracking-tutorials')
    # Hacking & Cracking questions
    # links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Hacking-Cracking-questions')
    # Exploit PoCs
    # links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Exploit-PoCs')
    # Cracked software
    # links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Cracked-software')
    # Malware-development
    # links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Malware-development')
    # Carding & Fraud
    # links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Carding-Fraud')
    # Darknet Discussions
    #links.append('http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/forumdisplay.php?fid=88')
    # OPSEC
    # links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-OPSEC')
    # Databases
    # links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Databases')
    # Proxies
    # links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Proxies')
    return links
 def crawlForum(driver):
    print("Crawling the OnniForums forum")
    linksToCrawl = getInterestedLinks()
    visited = set(linksToCrawl)
    initialTime = time.time()
    i = 0
    count = 0
    while i < len(linksToCrawl):
        link = linksToCrawl[i]
        print('Crawling :', link)
        try:
            try:
                driver.get(link)
            except:
                driver.refresh()
            html = driver.page_source
            savePage(html, link)
            has_next_page = True
            while has_next_page:
                list = topicPages(html)
                for item in list:
                    itemURL = urlparse.urljoin(baseURL, str(item))
                    try:
                        driver.get(itemURL)
                    except:
                        driver.refresh()
                    savePage(driver.page_source, item)
                    driver.back()
                    # comment out
                    break
                # comment out
                if count == 1:
                    count = 0
                    break
                try:
                    temp = driver.find_element(by=By.XPATH, value=
                        '/html/body/div/div[2]/div/div[2]/div')
                    link = temp.find_element(by=By.CLASS_NAME, value='pagination_next').get_attribute('href')
                    if link == "":
                        raise NoSuchElementException
                    try:
                        driver.get(link)
                    except:
                        driver.refresh()
                    html = driver.page_source
                    savePage(html, link)
                    count += 1
                except NoSuchElementException:
                    has_next_page = False
        except Exception as e:
            print(link, e)
        i += 1
    # finalTime = time.time()
    # print finalTime - initialTime
    input("Crawling OnniForums forum done sucessfully. Press ENTER to continue\n")
 # Returns 'True' if the link is Topic link
 def isDescriptionLink(url):
    if 'Thread' in url:
        return True
    return False
 # Returns True if the link is a listingPage link
 def isListingLink(url):
    if 'Forum' in url:
        return True
    return False
 # calling the parser to define the links
 def topicPages(html):
    soup = BeautifulSoup(html, "html.parser")
    #print(soup.find('div', id="container").find('div', id="content").find('table', {"class": "tborder clear"}).find('tbody').find('tr',{"class": "inline_row"}).find('strong').text)
    return cryptBB_links_parser(soup)
 def crawler():
    startCrawling()
    # print("Crawling and Parsing BestCardingWorld .... DONE!")
--- a/Forums/OnniForums/geckodriver.log
+++ b/Forums/OnniForums/geckodriver.log
@ -0,0 +1,462 @@
 1655762162153	geckodriver	INFO	Listening on 127.0.0.1:50333
 1655762166434	mozrunner::runner	INFO	Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50334" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileqUeAAN"
 console.log: "TorSettings: loadFromPrefs()"
 console.log: "TorConnect: Init"
 console.log: "TorConnect: observed profile-after-change"
 console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
 console.log: "TorConnect: observing topic 'TorBootstrapError'"
 console.log: "TorConnect: observing topic 'TorProcessExited'"
 console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
 console.log: "TorConnect: observing topic 'torsettings:ready'"
 console.log: "TorSettings: observed profile-after-change"
 1655762166904	Marionette	INFO	Marionette enabled
 console.log: "TorConnect: will load after bootstrap => [about:blank]"
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 DevTools listening on ws://localhost:50334/devtools/browser/fb880205-35da-44a4-83b8-a861ce7125f1
 1655762168590	Marionette	INFO	Listening on port 50341
 1655762169104	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
 1655762207624	Marionette	INFO	Stopped listening on port 50341
 JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
 !!! error running onStopped callback: TypeError: callback is not a function
 ###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost
 1655762207923	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
 unwatchForTabs()@TargetList.jsm:70
 unwatchForTargets()@TargetList.jsm:37
 destructor()@TargetList.jsm:109
 stop()@CDP.jsm:104
 close()@RemoteAgent.jsm:138
 1655762990320	geckodriver	INFO	Listening on 127.0.0.1:50884
 1655762994595	mozrunner::runner	INFO	Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50885" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileurjxEe"
 console.log: "TorSettings: loadFromPrefs()"
 console.log: "TorConnect: Init"
 console.log: "TorConnect: observed profile-after-change"
 console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
 console.log: "TorConnect: observing topic 'TorBootstrapError'"
 console.log: "TorConnect: observing topic 'TorProcessExited'"
 console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
 console.log: "TorConnect: observing topic 'torsettings:ready'"
 console.log: "TorSettings: observed profile-after-change"
 1655762995244	Marionette	INFO	Marionette enabled
 console.log: "TorConnect: will load after bootstrap => [about:blank]"
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 DevTools listening on ws://localhost:50885/devtools/browser/36503f58-7ab7-4188-81b0-fa1b81f32c0a
 1655762996997	Marionette	INFO	Listening on port 50890
 JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object.
 1655762997277	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
 1655763083091	Marionette	INFO	Stopped listening on port 50890
 JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
 !!! error running onStopped callback: TypeError: callback is not a function
 1655763083216	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
 unwatchForTabs()@TargetList.jsm:70
 unwatchForTargets()@TargetList.jsm:37
 destructor()@TargetList.jsm:109
 stop()@CDP.jsm:104
 close()@RemoteAgent.jsm:138
 1655763089557	geckodriver	INFO	Listening on 127.0.0.1:50923
 1655763093759	mozrunner::runner	INFO	Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50924" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile5mY1qq"
 console.log: "TorSettings: loadFromPrefs()"
 console.log: "TorConnect: Init"
 console.log: "TorConnect: observed profile-after-change"
 console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
 console.log: "TorConnect: observing topic 'TorBootstrapError'"
 console.log: "TorConnect: observing topic 'TorProcessExited'"
 console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
 console.log: "TorConnect: observing topic 'torsettings:ready'"
 console.log: "TorSettings: observed profile-after-change"
 1655763094208	Marionette	INFO	Marionette enabled
 console.log: "TorConnect: will load after bootstrap => [about:blank]"
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 DevTools listening on ws://localhost:50924/devtools/browser/68ce2df9-521f-4258-94f7-c2e2b199701a
 1655763095918	Marionette	INFO	Listening on port 50929
 1655763096408	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
 1655763143935	Marionette	INFO	Stopped listening on port 50929
 JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
 !!! error running onStopped callback: TypeError: callback is not a function
 1655763144495	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
 unwatchForTabs()@TargetList.jsm:70
 unwatchForTargets()@TargetList.jsm:37
 destructor()@TargetList.jsm:109
 stop()@CDP.jsm:104
 close()@RemoteAgent.jsm:138
 1655763166975	geckodriver	INFO	Listening on 127.0.0.1:50947
 1655763171175	mozrunner::runner	INFO	Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50948" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilenAGZM5"
 console.log: "TorSettings: loadFromPrefs()"
 console.log: "TorConnect: Init"
 console.log: "TorConnect: observed profile-after-change"
 console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
 console.log: "TorConnect: observing topic 'TorBootstrapError'"
 console.log: "TorConnect: observing topic 'TorProcessExited'"
 console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
 console.log: "TorConnect: observing topic 'torsettings:ready'"
 console.log: "TorSettings: observed profile-after-change"
 1655763171719	Marionette	INFO	Marionette enabled
 console.log: "TorConnect: will load after bootstrap => [about:blank]"
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 DevTools listening on ws://localhost:50948/devtools/browser/0b9966bd-0cec-45cc-9d11-02a2580233f0
 1655763173307	Marionette	INFO	Listening on port 50953
 1655763173368	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
 1655763208922	Marionette	INFO	Stopped listening on port 50953
 JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
 JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData]
 !!! error running onStopped callback: TypeError: callback is not a function
 1655763209261	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
 unwatchForTabs()@TargetList.jsm:70
 unwatchForTargets()@TargetList.jsm:37
 destructor()@TargetList.jsm:109
 stop()@CDP.jsm:104
 close()@RemoteAgent.jsm:138
 1655763387409	geckodriver	INFO	Listening on 127.0.0.1:49752
 1655763391806	mozrunner::runner	INFO	Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "49753" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilevAHJSJ"
 console.log: "TorSettings: loadFromPrefs()"
 console.log: "TorConnect: Init"
 console.log: "TorConnect: observed profile-after-change"
 console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
 console.log: "TorConnect: observing topic 'TorBootstrapError'"
 console.log: "TorConnect: observing topic 'TorProcessExited'"
 console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
 console.log: "TorConnect: observing topic 'torsettings:ready'"
 console.log: "TorSettings: observed profile-after-change"
 1655763392827	Marionette	INFO	Marionette enabled
 console.log: "TorConnect: will load after bootstrap => [about:blank]"
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
 DevTools listening on ws://localhost:49753/devtools/browser/7fe49a16-940f-4ee8-9a2f-48739a92db78
 1655763394700	Marionette	INFO	Listening on port 49759
 1655763395103	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
 1655763430848	Marionette	INFO	Stopped listening on port 49759
 JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
 JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData]
 !!! error running onStopped callback: TypeError: callback is not a function
 1655763431145	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
 unwatchForTabs()@TargetList.jsm:70
 unwatchForTargets()@TargetList.jsm:37
 destructor()@TargetList.jsm:109
 stop()@CDP.jsm:104
 close()@RemoteAgent.jsm:138
 1655765589286	geckodriver	INFO	Listening on 127.0.0.1:50102
 1655765621677	mozrunner::runner	INFO	Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50103" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile8ecUpb"
 console.log: "TorSettings: loadFromPrefs()"
 console.log: "TorConnect: Init"
 console.log: "TorConnect: observed profile-after-change"
 console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
 console.log: "TorConnect: observing topic 'TorBootstrapError'"
 console.log: "TorConnect: observing topic 'TorProcessExited'"
 console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
 console.log: "TorConnect: observing topic 'torsettings:ready'"
 console.log: "TorSettings: observed profile-after-change"
 1655765622149	Marionette	INFO	Marionette enabled
 console.log: "TorConnect: will load after bootstrap => [about:blank]"
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
 DevTools listening on ws://localhost:50103/devtools/browser/e5a916fc-987c-43d4-9c12-9c9b88cca242
 1655765623776	Marionette	INFO	Listening on port 50109
 1655765623892	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
 1655766125168	Marionette	INFO	Stopped listening on port 50109
 JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
 !!! error running onStopped callback: TypeError: callback is not a function
 1655766125303	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
 unwatchForTabs()@TargetList.jsm:70
 unwatchForTargets()@TargetList.jsm:37
 destructor()@TargetList.jsm:109
 stop()@CDP.jsm:104
 close()@RemoteAgent.jsm:138
 1655766505099	geckodriver	INFO	Listening on 127.0.0.1:50156
 1655766509351	mozrunner::runner	INFO	Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50157" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileVqvmzh"
 console.log: "TorSettings: loadFromPrefs()"
 console.log: "TorConnect: Init"
 console.log: "TorConnect: observed profile-after-change"
 console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
 console.log: "TorConnect: observing topic 'TorBootstrapError'"
 console.log: "TorConnect: observing topic 'TorProcessExited'"
 console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
 console.log: "TorConnect: observing topic 'torsettings:ready'"
 console.log: "TorSettings: observed profile-after-change"
 1655766509876	Marionette	INFO	Marionette enabled
 console.log: "TorConnect: will load after bootstrap => [about:blank]"
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 DevTools listening on ws://localhost:50157/devtools/browser/13084c14-64d3-48a5-8b99-4c514e961d4f
 1655766511328	Marionette	INFO	Listening on port 50162
 1655766511395	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
 1655766552298	Marionette	INFO	Stopped listening on port 50162
 JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
 JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData]
 !!! error running onStopped callback: TypeError: callback is not a function
 1655766552578	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
 unwatchForTabs()@TargetList.jsm:70
 unwatchForTargets()@TargetList.jsm:37
 destructor()@TargetList.jsm:109
 stop()@CDP.jsm:104
 close()@RemoteAgent.jsm:138
 1655767657126	geckodriver	INFO	Listening on 127.0.0.1:50257
 1655767661375	mozrunner::runner	INFO	Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50258" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileB9Dzeh"
 console.log: "TorSettings: loadFromPrefs()"
 console.log: "TorConnect: Init"
 console.log: "TorConnect: observed profile-after-change"
 console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
 console.log: "TorConnect: observing topic 'TorBootstrapError'"
 console.log: "TorConnect: observing topic 'TorProcessExited'"
 console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
 console.log: "TorConnect: observing topic 'torsettings:ready'"
 console.log: "TorSettings: observed profile-after-change"
 1655767661843	Marionette	INFO	Marionette enabled
 console.log: "TorConnect: will load after bootstrap => [about:blank]"
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 DevTools listening on ws://localhost:50258/devtools/browser/867cd318-6430-4b3f-9b63-8c6bdf17636a
 1655767663330	Marionette	INFO	Listening on port 50263
 1655767663431	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
 1655767668263	Marionette	INFO	Stopped listening on port 50263
 JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
 !!! error running onStopped callback: TypeError: callback is not a function
 ###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost
 1655767668591	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
 unwatchForTabs()@TargetList.jsm:70
 unwatchForTargets()@TargetList.jsm:37
 destructor()@TargetList.jsm:109
 stop()@CDP.jsm:104
 close()@RemoteAgent.jsm:138
 1655767689480	geckodriver	INFO	Listening on 127.0.0.1:50276
 1655767693635	mozrunner::runner	INFO	Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50277" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile8ZOQe7"
 console.log: "TorSettings: loadFromPrefs()"
 console.log: "TorConnect: Init"
 console.log: "TorConnect: observed profile-after-change"
 console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
 console.log: "TorConnect: observing topic 'TorBootstrapError'"
 console.log: "TorConnect: observing topic 'TorProcessExited'"
 console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
 console.log: "TorConnect: observing topic 'torsettings:ready'"
 console.log: "TorSettings: observed profile-after-change"
 1655767694215	Marionette	INFO	Marionette enabled
 console.log: "TorConnect: will load after bootstrap => [about:blank]"
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 DevTools listening on ws://localhost:50277/devtools/browser/a70a3f5b-19af-4a68-99c8-c46086ba9599
 1655767695725	Marionette	INFO	Listening on port 50282
 1655767695830	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
 1655767701272	Marionette	INFO	Stopped listening on port 50282
 JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
 !!! error running onStopped callback: TypeError: callback is not a function
 1655767701594	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
 unwatchForTabs()@TargetList.jsm:70
 unwatchForTargets()@TargetList.jsm:37
 destructor()@TargetList.jsm:109
 stop()@CDP.jsm:104
 close()@RemoteAgent.jsm:138
 1655767791360	geckodriver	INFO	Listening on 127.0.0.1:50296
 1655767795575	mozrunner::runner	INFO	Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50297" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile4GEqBr"
 console.log: "TorSettings: loadFromPrefs()"
 console.log: "TorConnect: Init"
 console.log: "TorConnect: observed profile-after-change"
 console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
 console.log: "TorConnect: observing topic 'TorBootstrapError'"
 console.log: "TorConnect: observing topic 'TorProcessExited'"
 console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
 console.log: "TorConnect: observing topic 'torsettings:ready'"
 console.log: "TorSettings: observed profile-after-change"
 1655767796001	Marionette	INFO	Marionette enabled
 console.log: "TorConnect: will load after bootstrap => [about:blank]"
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 DevTools listening on ws://localhost:50297/devtools/browser/e2d4e57a-16ce-429d-9804-e9e391199ddf
 1655767797534	Marionette	INFO	Listening on port 50302
 1655767797661	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
 1655767802833	Marionette	INFO	Stopped listening on port 50302
 JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
 !!! error running onStopped callback: TypeError: callback is not a function
 1655767803151	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
 unwatchForTabs()@TargetList.jsm:70
 unwatchForTargets()@TargetList.jsm:37
 destructor()@TargetList.jsm:109
 stop()@CDP.jsm:104
 close()@RemoteAgent.jsm:138
 1655767902908	geckodriver	INFO	Listening on 127.0.0.1:50320
 1655767907009	mozrunner::runner	INFO	Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50321" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileqlJ8ZA"
 console.log: "TorSettings: loadFromPrefs()"
 console.log: "TorConnect: Init"
 console.log: "TorConnect: observed profile-after-change"
 console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
 console.log: "TorConnect: observing topic 'TorBootstrapError'"
 console.log: "TorConnect: observing topic 'TorProcessExited'"
 console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
 console.log: "TorConnect: observing topic 'torsettings:ready'"
 console.log: "TorSettings: observed profile-after-change"
 1655767907484	Marionette	INFO	Marionette enabled
 console.log: "TorConnect: will load after bootstrap => [about:blank]"
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
 DevTools listening on ws://localhost:50321/devtools/browser/49e77669-01dd-447e-a804-36d42b3400cf
 1655767908782	Marionette	INFO	Listening on port 50326
 1655767909069	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
 1655767916218	Marionette	INFO	Stopped listening on port 50326
 JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
 !!! error running onStopped callback: TypeError: callback is not a function
 1655767916523	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
 unwatchForTabs()@TargetList.jsm:70
 unwatchForTargets()@TargetList.jsm:37
 destructor()@TargetList.jsm:109
 stop()@CDP.jsm:104
 close()@RemoteAgent.jsm:138
 1655767958348	geckodriver	INFO	Listening on 127.0.0.1:50340
 1655767962557	mozrunner::runner	INFO	Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50341" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilekpCdfs"
 console.log: "TorSettings: loadFromPrefs()"
 console.log: "TorConnect: Init"
 console.log: "TorConnect: observed profile-after-change"
 console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
 console.log: "TorConnect: observing topic 'TorBootstrapError'"
 console.log: "TorConnect: observing topic 'TorProcessExited'"
 console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
 console.log: "TorConnect: observing topic 'torsettings:ready'"
 console.log: "TorSettings: observed profile-after-change"
 1655767963032	Marionette	INFO	Marionette enabled
 console.log: "TorConnect: will load after bootstrap => [about:blank]"
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 DevTools listening on ws://localhost:50341/devtools/browser/b4d1728c-d26f-4537-adc2-5c954b171b13
 1655767964591	Marionette	INFO	Listening on port 50346
 1655767964633	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
 1655767973210	Marionette	INFO	Stopped listening on port 50346
 JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
 !!! error running onStopped callback: TypeError: callback is not a function
 1655767973501	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
 unwatchForTabs()@TargetList.jsm:70
 unwatchForTargets()@TargetList.jsm:37
 destructor()@TargetList.jsm:109
 stop()@CDP.jsm:104
 close()@RemoteAgent.jsm:138
 1655767998084	geckodriver	INFO	Listening on 127.0.0.1:50358
 1655768002314	mozrunner::runner	INFO	Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50359" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileujYNBj"
 console.log: "TorSettings: loadFromPrefs()"
 console.log: "TorConnect: Init"
 console.log: "TorConnect: observed profile-after-change"
 console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
 console.log: "TorConnect: observing topic 'TorBootstrapError'"
 console.log: "TorConnect: observing topic 'TorProcessExited'"
 console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
 console.log: "TorConnect: observing topic 'torsettings:ready'"
 console.log: "TorSettings: observed profile-after-change"
 1655768002757	Marionette	INFO	Marionette enabled
 console.log: "TorConnect: will load after bootstrap => [about:blank]"
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 DevTools listening on ws://localhost:50359/devtools/browser/04112c45-204d-43f1-9615-34782fd06632
 1655768004165	Marionette	INFO	Listening on port 50364
 1655768004375	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
 1655768009787	Marionette	INFO	Stopped listening on port 50364
 JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
 ###!!! [Child][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost
 !!! error running onStopped callback: TypeError: callback is not a function
 ###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost
 1655768010100	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
 unwatchForTabs()@TargetList.jsm:70
 unwatchForTargets()@TargetList.jsm:37
 destructor()@TargetList.jsm:109
 stop()@CDP.jsm:104
 close()@RemoteAgent.jsm:138
 1655768083966	geckodriver	INFO	Listening on 127.0.0.1:50386
 1655768088234	mozrunner::runner	INFO	Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50387" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile39Pl1f"
 console.log: "TorSettings: loadFromPrefs()"
 console.log: "TorConnect: Init"
 console.log: "TorConnect: observed profile-after-change"
 console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
 console.log: "TorConnect: observing topic 'TorBootstrapError'"
 console.log: "TorConnect: observing topic 'TorProcessExited'"
 console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
 console.log: "TorConnect: observing topic 'torsettings:ready'"
 console.log: "TorSettings: observed profile-after-change"
 1655768088664	Marionette	INFO	Marionette enabled
 console.log: "TorConnect: will load after bootstrap => [about:blank]"
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 DevTools listening on ws://localhost:50387/devtools/browser/3d6cda42-cb60-4c67-8d76-1d186a51887f
 1655768090263	Marionette	INFO	Listening on port 50392
 1655768090299	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
 1655768091655	Marionette	INFO	Stopped listening on port 50392
 JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
 !!! error running onStopped callback: TypeError: callback is not a function
 ###!!! [Parent][MessageChannel] Error: (msgtype=0x390076,name=PContent::Msg_DestroyBrowsingContextGroup) Closed channel: cannot send/recv
 ###!!! [Parent][MessageChannel] Error: (msgtype=0x390076,name=PContent::Msg_DestroyBrowsingContextGroup) Closed channel: cannot send/recv
 ###!!! [Parent][MessageChannel] Error: (msgtype=0x390076,name=PContent::Msg_DestroyBrowsingContextGroup) Closed channel: cannot send/recv
 1655768092659	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
 unwatchForTabs()@TargetList.jsm:70
 unwatchForTargets()@TargetList.jsm:37
 destructor()@TargetList.jsm:109
 stop()@CDP.jsm:104
 close()@RemoteAgent.jsm:138
 1655768101201	geckodriver	INFO	Listening on 127.0.0.1:50404
 1655768105373	mozrunner::runner	INFO	Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50405" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile2YRGSz"
 console.log: "TorSettings: loadFromPrefs()"
 console.log: "TorConnect: Init"
 console.log: "TorConnect: observed profile-after-change"
 console.log: "TorConnect: observing topic 'TorBootstrapStatus'"
 console.log: "TorConnect: observing topic 'TorBootstrapError'"
 console.log: "TorConnect: observing topic 'TorProcessExited'"
 console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'"
 console.log: "TorConnect: observing topic 'torsettings:ready'"
 console.log: "TorSettings: observed profile-after-change"
 1655768105853	Marionette	INFO	Marionette enabled
 console.log: "TorConnect: will load after bootstrap => [about:blank]"
 JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
 console.error: "Could not load engine [email protected]: Error: Extension is invalid"
 DevTools listening on ws://localhost:50405/devtools/browser/9c4a4d71-25ff-4980-a54d-8545d6200790
 1655768107431	Marionette	INFO	Listening on port 50410
 1655768107530	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
 1655768119155	Marionette	INFO	Stopped listening on port 50410
 JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
 !!! error running onStopped callback: TypeError: callback is not a function
 1655768119438	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
 unwatchForTabs()@TargetList.jsm:70
 unwatchForTargets()@TargetList.jsm:37
 destructor()@TargetList.jsm:109
 stop()@CDP.jsm:104
 close()@RemoteAgent.jsm:138
 1655768229472	geckodriver	INFO	Listening on 127.0.0.1:50431
 1655768233458	webdriver::server	WARN	Rejected request with missing Host header
--- a/Forums/OnniForums/parser.py
+++ b/Forums/OnniForums/parser.py
@ -0,0 +1,354 @@
 __author__ = 'DarkWeb'
 # Here, we are importing the auxiliary functions to clean or convert data
 from Forums.Utilities.utilities import *
 from datetime import date
 from datetime import timedelta
 import re
 # Here, we are importing BeautifulSoup to search through the HTML tree
 from bs4 import BeautifulSoup
 # This is the method to parse the Description Pages (one page to each topic in the Listing Pages)
 def cryptBB_description_parser(soup):
    # Fields to be parsed
    topic = "-1"           # topic name
    user = []              # all users of each post
    addDate = []           # all dated of each post
    feedback = []          # all feedbacks of each vendor (this was found in just one Forum and with a number format)
    status = []            # all user's authority in each post such as (adm, member, dangerous)
    reputation = []        # all users's karma in each post (usually found as a number)
    sign = []              # all user's signature in each post (usually a standard message after the content of the post)
    post = []              # all messages of each post
    interest = []          # all user's interest in each post
    # Finding the topic (should be just one coming from the Listing Page)
    li = soup.find("td", {"class": "thead"}).find('strong')
    topic = li.text
    topic = re.sub("\[\w*\]", '', topic)
    topic = topic.replace(",","")
    topic = topic.replace("\n","")
    topic = cleanString(topic.strip())
    print(topic)
    # Finding the repeated tag that corresponds to the listing of posts
    # posts = soup.find("form", {"name": "quickModForm"}).findAll('div', {"class": "windowbg"}) + \
    #         soup.find("form", {"name": "quickModForm"}).findAll('div', {"class": "windowbg2"})
    try:
        posts = soup.find('table', {"class": "tborder tfixed clear"}).find('td', {"id": "posts_container"}).find_all(
            'div', {"class": "post"})
        # print(len(posts))
        # For each message (post), get all the fields we are interested to:
        for ipost in posts:
            # Finding a first level of the HTML page
            # post_wrapper = ipost.find('div', {"class": "post_wrapper"}).find('div', {"class": "poster"})
            post_wrapper = ipost.find('span', {"class": "largetext"})
            # Finding the author (user) of the post
            # author = post_wrapper.find('h4')
            author = post_wrapper.text.strip()
            # print("author " + author)
            user.append(cleanString(author))  # Remember to clean the problematic characters
            # Finding the status of the author
            smalltext = ipost.find('div', {"class": "post_author"})
            # Testing here two possibilities to find this status and combine them
            if ipost.find('div', {"class": "deleted_post_author"}):
                status.append(-1)
                interest.append(-1)
                reputation.append(-1)
                addDate.append(-1)
                post.append("THIS POST HAS BEEN REMOVED!")
                sign.append(-1)
                feedback.append(-1)
                continue
            # CryptBB does have membergroup and postgroup
            membergroup = smalltext.find('div', {"class": "profile-rank"})
            postgroup = smalltext.find('div', {"class": "postgroup"})
            if membergroup != None:
                membergroup = membergroup.text.strip()
                if postgroup != None:
                    postgroup = postgroup.text.strip()
                    membergroup = membergroup + " - " + postgroup
            else:
                if postgroup != None:
                    membergroup = postgroup.text.strip()
                else:
                    membergroup = "-1"
            status.append(cleanString(membergroup))
            # print("status " + cleanString(membergroup))
            # Finding the interest of the author
            # CryptBB does not have blurb
            blurb = smalltext.find('li', {"class": "blurb"})
            if blurb != None:
                blurb = blurb.text.strip()
            else:
                blurb = "-1"
            interest.append(cleanString(blurb))
            # Finding the reputation of the user
            # CryptBB does have reputation
            author_stats = smalltext.find('div', {"class": "author_statistics"})
            karma = author_stats.find('strong')
            if karma != None:
                karma = karma.text
                karma = karma.replace("Community Rating: ", "")
                karma = karma.replace("Karma: ", "")
                karma = karma.strip()
            else:
                karma = "-1"
            reputation.append(cleanString(karma))
            # print("karma " + cleanString(karma))
            # Getting here another good tag to find the post date, post content and users' signature
            postarea = ipost.find('div', {"class": "post_content"})
            dt = postarea.find('span', {"class": "post_date"}).text
            # dt = dt.strip().split()
            dt = dt.strip()
            day=date.today()
            if "Yesterday" in dt:
                yesterday = day - timedelta(days=1)
                yesterday = yesterday.strftime('%m-%d-%Y')
                stime = dt.replace('Yesterday,','').strip()
                date_time_obj = yesterday+ ', '+stime
                date_time_obj = datetime.strptime(date_time_obj,'%m-%d-%Y, %I:%M %p')
            elif "hours ago" in dt:
                day = day.strftime('%m-%d-%Y')
                date_time_obj = postarea.find('span', {"class": "post_date"}).find('span')['title']
                date_time_obj = datetime.strptime(date_time_obj, '%m-%d-%Y, %I:%M %p')
            else:
                date_time_obj = datetime.strptime(dt, '%m-%d-%Y, %I:%M %p')
                stime = date_time_obj.strftime('%b %d, %Y')
                sdate = date_time_obj.strftime('%I:%M %p')
            addDate.append(date_time_obj)
            # print("date " + str(date_time_obj))
            # Finding the date of the post
            # date_time_obj = datetime.strptime(dt, '%a %b %d, %Y %I:%M %p')
            # smalltext = postarea.find('div', {"class": "flow_hidden"}).find('div', {"class": "keyinfo"})\
            #     .find('div', {"class": "smalltext"})
            # sdatetime = smalltext.text
            # sdatetime = sdatetime.replace(u"\xab","") # Removing unnecessary characters
            # sdatetime = sdatetime.replace(u"\xbb","") # Removing unnecessary characters
            # sdatetime = sdatetime.split("on: ")       # Removing unnecessary characters
            # sdatetime = sdatetime[1].strip()
            # stime = sdatetime[:-12:-1]                # Finding the time of the post
            # stime = stime[::-1]
            # sdate = sdatetime.replace(stime,"")       # Finding the date of the post
            # sdate = sdate.replace(",","")
            # sdate = sdate.strip()
            # Covert the date of the post that can be informed as: "12 February 2016", "today", "yesterday". We need
            # a date format here as "mm/dd/yyyy"
            # addDate.append(convertDate(sdate,"english", crawlerDate) + " " + stime)
            # Finding the post
            inner = postarea.find('div', {"class": "post_body scaleimages"})
            inner = inner.text.strip()
            # print(inner)
            post.append(cleanString(inner))
            # Finding the users's signature
            # signature = ipost.find('div', {"class": "post_wrapper"}).find('div', {"class": "moderatorbar"}).find('div', {"class": "signature"})
            signature = ipost.find('div', {"class": "signature scaleimages"})
            if signature != None:
                signature = signature.text.strip()
                # print(signature)
            else:
                signature = "-1"
            sign.append(cleanString(signature))
            # As no information about users's feedback was found, just assign "-1" to the variable
            feedback.append("-1")
    except:
        if soup.find('td', {"class": "trow1"}).text == " You do not have permission to access this page. ":
            user.append("-1")
            status.append(-1)
            interest.append(-1)
            reputation.append(-1)
            addDate.append(-1)
            post.append("NO ACCESS TO THIS PAGE!")
            sign.append(-1)
            feedback.append(-1)
    # Populate the final variable (this should be a list with all fields scraped)
    row = (topic, post, user, addDate, feedback, status, reputation, sign, interest)
    # Sending the results
    return row
 # This is the method to parse the Listing Pages (one page with many posts)
 def cryptBB_listing_parser(soup):
    board = "-1"       # board name (the previous level of the topic in the Forum categorization tree.
                       # For instance: Security/Malware/Tools to hack Facebook. The board here should be Malware)
    nm = 0             # this variable should receive the number of topics
    topic = []         # all topics
    user = []          # all users of each topic
    post = []          # number of posts of each topic
    view = []          # number of views of each topic
    addDate = []       # when the topic was created (difficult to find)
    href = []          # this variable should receive all cleaned urls (we will use this to do the marge between
                       # Listing and Description pages)
    # Finding the board (should be just one)
    board = soup.find('span', {"class": "active"}).text
    board = cleanString(board.strip())
    # Finding the repeated tag that corresponds to the listing of topics
    itopics = soup.find_all('tr',{"class": "inline_row"})
    index = 0
    for itopic in itopics:
        # For each topic found, the structure to get the rest of the information can be of two types. Testing all of them
        # to don't miss any topic
        # Adding the topic to the topic list
        try:
            topics = itopic.find('span', {"class": "subject_old"}).find('a').text
        except:
            topics = itopic.find('span', {"class": "subject_new"}).find('a').text
        topics = re.sub("\[\w*\]", '', topics)
        topic.append(cleanString(topics))
        # Counting how many topics we have found so far
        nm = len(topic)
        # Adding the url to the list of urls
        try:
            link = itopic.find('span', {"class": "subject_old"}).find('a').get('href')
        except:
            link = itopic.find('span',{"class": "subject_new"}).find('a').get('href')
        link = cleanLink(link)
        href.append(link)
        # Finding the author of the topic
        ps = itopic.find('div', {"class":"author smalltext"}).find('a').text
        author = ps.strip()
        user.append(cleanString(author))
        # Finding the number of replies
        columns = itopic.findChildren('td',recursive=False)
        posts = columns[3].text
        post.append(cleanString(posts))
        # Finding the number of Views
        tview = columns[4].text
        view.append(cleanString(tview))
        # If no information about when the topic was added, just assign "-1" to the variable
        #dt = itopic.find('div', {"class": "responsive-hide"}).text.split('»')[1]
        #dt = dt.strip()
        #date_time_obj = datetime.strptime(dt,'%a %b %d, %Y %I:%M %p')
        #addDate.append(date_time_obj)
        addDate.append("-1")
        index += 1
    return organizeTopics("CryptBB", nm, topic, board, view, post, user, addDate, href)
        # if len(tag) > 0:
        #
        #     # Finding the topic
        #
        #     tds = tds[0].find(tag[0])
        #     topics = tds.text
        #     topics = topics.replace(u"\xbb","")
        #     topics = topics.strip()
        #     topic.append(cleanString(topics))
        #
        #     # Counting how many topics we have found so far
        #
        #     nm = len(topic)
        #
        #     # Adding the url to the list of urls
        #
        #     link = tds.findAll('a', href=True)
        #     link = link[0].get('href')
        #     link = cleanLink(link)
        #     href.append(link)
        #
        #     # Finding the author of the topic
        #
        #     ps = itopic.find('td', {"class": tag[1]}).find('p').find('a')
        #     if ps == None:
        #        ps = itopic.find('td', {"class": tag[1]}).find('p')
        #        ps = ps.text.replace("Started by ","")
        #     else:
        #        ps = ps.text
        #     author = ps.strip()
        #     user.append(cleanString(author))
        #
        #     # Finding the number of replies
        #
        #     statistics = itopic.find('td', {"class": tag[2]})
        #     statistics = statistics.text
        #     statistics = statistics.split("Replies")
        #     posts = statistics[0].strip()
        #     post.append(cleanString(posts))
        #
        #     # Finding the number of Views
        #
        #     views = statistics[1]
        #     views = views.replace("Views","")
        #     views = views.strip()
        #     view.append(cleanString(views))
        #
        #     # As no information about when the topic was added, just assign "-1" to the variable
        #
        #     addDate.append("-1")
    #return organizeTopics("TheMajesticGarden", nm, topic, board, view, post, user, addDate, href)
 def cryptBB_links_parser(soup):
    # Returning all links that should be visited by the Crawler
    href = []
    #print(soup.find('table', {"class": "tborder clear"}).find(
     #   'tbody').find_all('tr', {"class": "inline_row"}))
    listing = soup.find('table', {"class": "tborder clear"}).find('tbody').find_all('tr', {"class": "inline_row"})
    for a in listing:
        try:
            link = a.find('span', {"class": "subject_old"}).find('a').get('href')
        except:
            link = a.find('span', {"class": "subject_new"}).find('a').get('href')
        href.append(link)
    return href
--- a/path.txt
+++ b/path.txt
@ -1,3 +1,3 @@
 C:\Users\calsyslab\Desktop\Tor Browser\Browser\firefox.exe
 C:\Users\calsyslab\Desktop\Tor Browser\Browser\TorBrowser\Data\Browser\profile.default
 C:\Users\calsyslab\Documents\CALSysLab\threatIntelligence\DarkWebMining_Working\selenium\geckodriver.exe
 C:\Users\Helium\Desktop\Tor Browser\Browser\firefox.exe
 C:\Users\Helium\Desktop\Tor Browser\Browser\TorBrowser\Data\Browser\profile.default
 C:\Users\Helium\PycharmProjects\dw_pipeline_test\selenium\geckodriver.exe