diff --git a/.idea/DW_Pipeline_Test.iml b/.idea/DW_Pipeline_Test.iml index 6a94dd3..71f5e9b 100644 --- a/.idea/DW_Pipeline_Test.iml +++ b/.idea/DW_Pipeline_Test.iml @@ -2,7 +2,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index 883bc94..baf04e9 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -1,4 +1,4 @@ - + \ No newline at end of file diff --git a/Forums/BestCardingWorld/crawler_selenium.py b/Forums/BestCardingWorld/crawler_selenium.py index e12b601..8eaa261 100644 --- a/Forums/BestCardingWorld/crawler_selenium.py +++ b/Forums/BestCardingWorld/crawler_selenium.py @@ -96,7 +96,7 @@ def createFFDriver(): ff_prof.set_preference("privacy.sanitize.sanitizeOnShutdown", True) ff_prof.set_preference("signon.rememberSignons", False) ff_prof.set_preference("network.cookie.lifetimePolicy", 2) - ff_prof.set_preference("network.dns.disablePrefetch", True) + ff_prof.set_preference("network.dns.disablePrefetch", True)#might need to turn off ff_prof.set_preference("network.http.sendRefererHeader", 0) ff_prof.set_preference("permissions.default.image", 2) ff_prof.set_preference("browser.download.folderList", 2) diff --git a/Forums/CryptBB/crawler_selenium.py b/Forums/CryptBB/crawler_selenium.py index 04d536c..e73b291 100644 --- a/Forums/CryptBB/crawler_selenium.py +++ b/Forums/CryptBB/crawler_selenium.py @@ -36,7 +36,7 @@ def startCrawling(): if driver != 'down': try: login(driver) - crawlForum(driver) + # crawlForum(driver) except Exception as e: print(driver.current_url, e) closetor(driver) @@ -144,7 +144,7 @@ def createFFDriver(): ff_prof.set_preference("privacy.sanitize.sanitizeOnShutdown", True) ff_prof.set_preference("signon.rememberSignons", False) ff_prof.set_preference("network.cookie.lifetimePolicy", 2) - ff_prof.set_preference("network.dns.disablePrefetch", True) + ff_prof.set_preference("network.dns.disablePrefetch", True)# ff_prof.set_preference("network.http.sendRefererHeader", 0) ff_prof.set_preference("permissions.default.image", 3) ff_prof.set_preference("browser.download.folderList", 2) diff --git a/Forums/Initialization/forumsList.txt b/Forums/Initialization/forumsList.txt index 855f481..ebdc9b3 100644 --- a/Forums/Initialization/forumsList.txt +++ b/Forums/Initialization/forumsList.txt @@ -1 +1 @@ -CryptBB +CryptBB \ No newline at end of file diff --git a/Forums/Initialization/forums_mining.py b/Forums/Initialization/forums_mining.py index 8aa03be..5a68c38 100644 --- a/Forums/Initialization/forums_mining.py +++ b/Forums/Initialization/forums_mining.py @@ -8,11 +8,12 @@ import os from datetime import * from Forums.BestCardingWorld.crawler_selenium import crawler as crawlerBestCardingWorld from Forums.CryptBB.crawler_selenium import crawler as crawlerCryptBB +from Forums.OnniForums.crawler_selenium import crawler as crawlerOnniForums import time -# reads list of marketplaces +# reads list of marketplaces manually inputted def getForums(): forums = [] with open('forumsList.txt') as f: @@ -77,22 +78,27 @@ def createSubdirectories(pagesDir): os.mkdir(descReadDir) +#main method if __name__ == '__main__': - print("HelloWorld") + + #assignment from forumsList.txt forumsList = getForums() + #get forum from forumsList for forum in forumsList: forum = forum.replace('\n','') print("Creating listing and description directories ...") createDirectory(forum) - time.sleep(5) + time.sleep(5) #wait for directories to be created input("Directories created successfully. Press ENTER to continue\n") if forum == "BestCardingWorld": crawlerBestCardingWorld() elif forum == "CryptBB": crawlerCryptBB() + elif forum == "OnniForums": + crawlerOnniForums() print("Scraping process completed successfully!") diff --git a/Forums/Initialization/geckodriver.log b/Forums/Initialization/geckodriver.log index aaafa73..7696b21 100644 --- a/Forums/Initialization/geckodriver.log +++ b/Forums/Initialization/geckodriver.log @@ -1402,3 +1402,310 @@ unwatchForTargets()@TargetList.jsm:37 destructor()@TargetList.jsm:109 stop()@CDP.jsm:104 close()@RemoteAgent.jsm:138 +1686338884789 geckodriver INFO Listening on 127.0.0.1:51464 +1686338889554 mozrunner::runner INFO Running command: "C:\\Users\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "51465" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofilePx8hdP" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: init()" +console.log: "TorConnect: Entering Initial state" +console.log: "TorConnect: Observed profile-after-change" +console.log: "TorConnect: Observing topic 'TorProcessExited'" +console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: Observing topic 'torsettings:ready'" +console.log: "TorSettings: Observed profile-after-change" +1686338895681 Marionette INFO Marionette enabled +console.log: "TorConnect: Will load after bootstrap => [about:blank]" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:51465/devtools/browser/ca518483-f04e-427e-89d6-a996307996a5 +1686338902642 Marionette INFO Listening on port 51470 +1686338902983 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/, line 2: ReferenceError: $ is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 2: ReferenceError: $ is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 5: ReferenceError: lang is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 9: ReferenceError: use_xmlhttprequest is not defined +1686338987650 Marionette INFO Stopped listening on port 51470 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +[Parent 5940, IPC I/O Parent] WARNING: file /var/tmp/build/firefox-b6010b1466c9/ipc/chromium/src/base/process_util_win.cc:167 +!!! error running onStopped callback: TypeError: callback is not a function + +###!!! [Child][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + +JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first. +JavaScript error: resource://gre/modules/PageThumbs.jsm, line 709: AbortError: IOUtils.profileBeforeChange getter: IOUtils: profileBeforeChange phase has already finished + +###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + +1686338988537 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1686596562082 geckodriver INFO Listening on 127.0.0.1:50187 +1686596566629 mozrunner::runner INFO Running command: "C:\\Users\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50188" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofileQhGGOx" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: init()" +console.log: "TorConnect: Entering Initial state" +console.log: "TorConnect: Observed profile-after-change" +console.log: "TorConnect: Observing topic 'TorProcessExited'" +console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: Observing topic 'torsettings:ready'" +console.log: "TorSettings: Observed profile-after-change" +1686596576118 Marionette INFO Marionette enabled +console.log: "TorConnect: Will load after bootstrap => [about:blank]" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +DevTools listening on ws://localhost:50188/devtools/browser/3324b32e-574b-4b12-8ad5-138d84a7b3b0 +1686596590433 Marionette INFO Listening on port 50204 +1686596590541 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/, line 2: ReferenceError: $ is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 2: ReferenceError: $ is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 5: ReferenceError: lang is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 9: ReferenceError: use_xmlhttprequest is not defined +1686596652376 Marionette INFO Stopped listening on port 50204 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first. +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofileQhGGOx\thumbnails) because it does not exist +1686596653163 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1686605562853 geckodriver INFO Listening on 127.0.0.1:50934 +1686605574898 mozrunner::runner INFO Running command: "C:\\Users\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50935" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofileDnVqxJ" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: init()" +console.log: "TorConnect: Entering Initial state" +console.log: "TorConnect: Observed profile-after-change" +console.log: "TorConnect: Observing topic 'TorProcessExited'" +console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: Observing topic 'torsettings:ready'" +console.log: "TorSettings: Observed profile-after-change" +1686605580062 Marionette INFO Marionette enabled +console.log: "TorConnect: Will load after bootstrap => [about:blank]" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:50935/devtools/browser/31caaf51-c751-4c17-b821-bbe5b1b5eb18 +1686605587267 Marionette INFO Listening on port 50964 +1686605587958 RemoteAgent WARN TLS certificate errors will be ignored for this session +[Parent 8804, IPC I/O Parent] WARNING: file /var/tmp/build/firefox-b6010b1466c9/ipc/chromium/src/base/process_util_win.cc:167 +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/, line 2: ReferenceError: $ is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 2: ReferenceError: $ is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 5: ReferenceError: lang is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 9: ReferenceError: use_xmlhttprequest is not defined +console.log: "TorConnect: beginBootstrap()" +console.log: "TorConnect: Try transitioning from Initial to Bootstrapping" +console.log: "TorConnect: Exited Initial state" +console.log: "TorConnect: Entering Bootstrapping state" +JavaScript error: resource://gre/actors/RemotePageChild.jsm, line 116: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIXPCComponents_Utils.cloneInto] +1686605733273 Marionette INFO Stopped listening on port 50964 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData] +!!! error running onStopped callback: TypeError: callback is not a function +JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first. +JavaScript error: resource://gre/modules/PageThumbs.jsm, line 709: AbortError: IOUtils.profileBeforeChange getter: IOUtils: profileBeforeChange phase has already finished +[Parent 8804, IPC I/O Parent] WARNING: pipe error: 232: file /var/tmp/build/firefox-b6010b1466c9/ipc/chromium/src/chrome/common/ipc_channel_win.cc:544 +1686605733655 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1686609198317 geckodriver INFO Listening on 127.0.0.1:51716 +1686609202661 mozrunner::runner INFO Running command: "C:\\Users\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "51717" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofilei8xuPT" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: init()" +console.log: "TorConnect: Entering Initial state" +console.log: "TorConnect: Observed profile-after-change" +console.log: "TorConnect: Observing topic 'TorProcessExited'" +console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: Observing topic 'torsettings:ready'" +console.log: "TorSettings: Observed profile-after-change" +1686609203765 Marionette INFO Marionette enabled +console.log: "TorConnect: Will load after bootstrap => [about:blank]" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:51717/devtools/browser/9102313b-d646-4ee2-a043-875c18e99d72 +1686609206596 Marionette INFO Listening on port 51725 +1686609207200 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/, line 3: ReferenceError: lang is not defined +JavaScript error: http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/, line 34: ReferenceError: $ is not defined +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofilei8xuPT\thumbnails) because it does not exist +1686609237384 Marionette INFO Stopped listening on port 51725 +JavaScript error: resource://gre/modules/AsyncShutdown.jsm, line 575: NotFoundError: No such JSWindowActor 'MarionetteEvents' +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first. +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofilei8xuPT\thumbnails) because it does not exist +1686609238028 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1686609367764 geckodriver INFO Listening on 127.0.0.1:51806 +1686609371554 mozrunner::runner INFO Running command: "C:\\Users\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "51807" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofilesBnBWf" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: init()" +console.log: "TorConnect: Entering Initial state" +console.log: "TorConnect: Observed profile-after-change" +console.log: "TorConnect: Observing topic 'TorProcessExited'" +console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: Observing topic 'torsettings:ready'" +console.log: "TorSettings: Observed profile-after-change" +1686609372356 Marionette INFO Marionette enabled +console.log: "TorConnect: Will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:51807/devtools/browser/f7991b13-8912-433b-8eea-642199ceedab +1686609375774 Marionette INFO Listening on port 51812 +1686609375994 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofilesBnBWf\thumbnails) because it does not exist +1686609438764 Marionette INFO Stopped listening on port 51812 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first. +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofilesBnBWf\thumbnails) because it does not exist +1686609439513 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1686609756680 geckodriver INFO Listening on 127.0.0.1:51911 +1686609760338 mozrunner::runner INFO Running command: "C:\\Users\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "51912" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofile9AutkK" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: init()" +console.log: "TorConnect: Entering Initial state" +console.log: "TorConnect: Observed profile-after-change" +console.log: "TorConnect: Observing topic 'TorProcessExited'" +console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: Observing topic 'torsettings:ready'" +console.log: "TorSettings: Observed profile-after-change" +1686609761075 Marionette INFO Marionette enabled +console.log: "TorConnect: Will load after bootstrap => [about:blank]" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:51912/devtools/browser/72ad40a2-a5be-4660-b604-94a3f50327fb +1686609763168 Marionette INFO Listening on port 51917 +1686609763658 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/, line 2: ReferenceError: $ is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 2: ReferenceError: $ is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 5: ReferenceError: lang is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 9: ReferenceError: use_xmlhttprequest is not defined +console.log: "TorConnect: beginBootstrap()" +console.log: "TorConnect: Try transitioning from Initial to Bootstrapping" +console.log: "TorConnect: Exited Initial state" +JavaScript error: resource:///modules/TorConnectParent.jsm, line 110: InvalidStateError: JSWindowActorParent.sendAsyncMessage: JSWindowActorParent cannot send at the moment +console.log: "TorConnect: Entering Bootstrapping state" +JavaScript error: resource://gre/actors/RemotePageChild.jsm, line 116: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIXPCComponents_Utils.cloneInto] +1686609860153 Marionette INFO Stopped listening on port 51917 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first. +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofile9AutkK\thumbnails) because it does not exist +[Parent 2244, IPC I/O Parent] WARNING: file /var/tmp/build/firefox-b6010b1466c9/ipc/chromium/src/base/process_util_win.cc:167 +1686609860787 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1686610746918 geckodriver INFO Listening on 127.0.0.1:52097 +1686610750204 mozrunner::runner INFO Running command: "C:\\Users\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "52098" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofileMks8y4" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: init()" +console.log: "TorConnect: Entering Initial state" +console.log: "TorConnect: Observed profile-after-change" +console.log: "TorConnect: Observing topic 'TorProcessExited'" +console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: Observing topic 'torsettings:ready'" +console.log: "TorSettings: Observed profile-after-change" +1686610751007 Marionette INFO Marionette enabled +console.log: "TorConnect: Will load after bootstrap => [about:blank]" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:52098/devtools/browser/57fa325f-b035-44b6-a179-7f6407f259a5 +1686610753158 Marionette INFO Listening on port 52103 +1686610753378 RemoteAgent WARN TLS certificate errors will be ignored for this session +1686610846274 Marionette INFO Stopped listening on port 52103 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first. +JavaScript error: resource://gre/modules/PageThumbs.jsm, line 709: AbortError: IOUtils.profileBeforeChange getter: IOUtils: profileBeforeChange phase has already finished +1686610846887 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1686611138064 geckodriver INFO Listening on 127.0.0.1:52201 +1686611140362 mozrunner::runner INFO Running command: "C:\\Users\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "52202" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofileCzAsf4" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: init()" +console.log: "TorConnect: Entering Initial state" +console.log: "TorConnect: Observed profile-after-change" +console.log: "TorConnect: Observing topic 'TorProcessExited'" +console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: Observing topic 'torsettings:ready'" +console.log: "TorSettings: Observed profile-after-change" +1686611140911 Marionette INFO Marionette enabled +console.log: "TorConnect: Will load after bootstrap => [about:blank]" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:52202/devtools/browser/fda7fcbe-469d-43ad-a4dd-5eebc6b30bc6 +1686611142583 Marionette INFO Listening on port 52207 +1686611142902 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/, line 2: ReferenceError: $ is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 2: ReferenceError: $ is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 5: ReferenceError: lang is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 9: ReferenceError: use_xmlhttprequest is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 2: ReferenceError: $ is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 5: ReferenceError: lang is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 9: ReferenceError: use_xmlhttprequest is not defined +1686611206885 Marionette INFO Stopped listening on port 52207 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first. +JavaScript error: resource://gre/modules/PageThumbs.jsm, line 709: AbortError: IOUtils.profileBeforeChange getter: IOUtils: profileBeforeChange phase has already finished +1686611207399 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 diff --git a/Forums/OnniForums/captcha.png b/Forums/OnniForums/captcha.png new file mode 100644 index 0000000..08e45fc Binary files /dev/null and b/Forums/OnniForums/captcha.png differ diff --git a/Forums/OnniForums/crawler_mechanize.py b/Forums/OnniForums/crawler_mechanize.py new file mode 100644 index 0000000..e6b5001 --- /dev/null +++ b/Forums/OnniForums/crawler_mechanize.py @@ -0,0 +1,257 @@ +__author__ = '91Shadows' + +''' +BestCardingWorld Crawler (Mechanize) +''' + +import codecs, os, re +import socks, socket, time +from datetime import date + +import urllib.parse as urlparse +import http.client as httplib +import mechanize +import subprocess +from bs4 import BeautifulSoup +from Forums.Initialization.prepare_parser import new_parse +from Forums.BestCardingWorld.parser import bestcardingworld_links_parser + +counter = 1 +httplib.HTTPConnection._http_vsn = 10 +httplib.HTTPConnection._http_vsn_str = 'HTTP/1.0' +baseURL = 'http://bestteermb42clir6ux7xm76d4jjodh3fpahjqgbddbmfrgp4skg2wqd.onion/viewforum.php?f=42&sid=ee2cbfd73c12923d979790b2bb4bdfd5' +socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, "127.0.0.1", 9150) + + +# Opens Tor Browser, crawls the website +def startCrawling(): + opentor() + getUrl() + forumName = getForumName() + br = getAccess() + + if br != 'down': + crawlForum(br) + new_parse(forumName, False) + + # new_parse(forumName, False) + + closetor() + + +# Opens Tor Browser +def opentor(): + global pid + print("Connecting Tor...") + path = open('../../path.txt').readline() + pro = subprocess.Popen(path) + pid = pro.pid + time.sleep(7.5) + input("Tor Connected. Press ENTER to continue\n") + return + + +# Creates a connection through Tor Port +def getUrl(timeout=None): + socket.socket = socks.socksocket + socket.create_connection = create_connection + return + + +# Makes the onion address request +def create_connection(address, timeout=None, source_address=None): + sock = socks.socksocket() + sock.connect(address) + return sock + + +# Returns the name of website +def getForumName(): + name = 'BestCardingWorld' + return name + + +# Return the link of website +def getFixedURL(): + url = 'http://bestteermb42clir6ux7xm76d4jjodh3fpahjqgbddbmfrgp4skg2wqd.onion/viewforum.php?f=42&sid=ee2cbfd73c12923d979790b2bb4bdfd5' + + return url + + +# Closes Tor Browser +def closetor(): + global pid + os.system("taskkill /pid " + str(pid)) + print('Closing Tor...') + time.sleep(3) + return + + +# Creates a Mechanize browser and initializes its options +def createBrowser(): + br = mechanize.Browser() + cj = mechanize.CookieJar() + br.set_cookiejar(cj) + + # Browser options + br.set_handle_equiv(True) + br.set_handle_redirect(True) + br.set_handle_referer(True) + br.set_handle_robots(False) + br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1) + br.addheaders = [('User-agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko'), + ('Accept', '*/*')] + + return br + + +def getAccess(): + url = getFixedURL() + br = createBrowser() + + try: + + br.open(url) + return br + + except: + + return 'down' + + +# Saves the crawled html page +def savePage(page, url): + filePath = getFullPathName(url) + os.makedirs(os.path.dirname(filePath), exist_ok=True) + a = page.read() + open(filePath, "wb").write(a) + return + + +# Gets the full path of the page to be saved along with its appropriate file name +def getFullPathName(url): + fileName = getNameFromURL(url) + if isDescriptionLink(url): + fullPath = 'C:/Users/CALSysLab/Documents/threatIntelligence-main/DarkWebMining_Working/Forums/BestCardingWorld/HTML_Pages/' + str( + "%02d" % date.today().month) + str("%02d" % date.today().day) + str( + "%04d" % date.today().year) + '/' + 'Description/' + fileName + '.html' + else: + fullPath = 'C:/Users/CALSysLab/Documents/threatIntelligence-main/DarkWebMining_Working/Forums/BestCardingWorld/HTML_Pages/' + str( + "%02d" % date.today().month) + str("%02d" % date.today().day) + str( + "%04d" % date.today().year) + '/' + 'Listing/' + fileName + '.html' + return fullPath + + +# Creates the name of the file based on URL +def getNameFromURL(url): + global counter + name = ''.join(e for e in url if e.isalnum()) + if (name == ''): + name = str(counter) + counter = counter + 1 + return name + + +# Hacking and Markets related topics +def getInterestedLinks(): + links = [] + + links.append('http://bestteermb42clir6ux7xm76d4jjodh3fpahjqgbddbmfrgp4skg2wqd.onion/viewforum.php?f=43&sid=e12864ffccc5df877b03b573534955be') + + return links + + +# Start crawling Forum pages +def crawlForum(br): + print("Crawling The Best Carding World forum") + + linksToCrawl = getInterestedLinks() + visited = set(linksToCrawl) + initialTime = time.time() + + + i = 0 + while i < len(linksToCrawl): + link = linksToCrawl[i] + print('Crawling :', link) + try: + page = br.open(link) + savePage(page, link) + + res = br.response().read() + soup = BeautifulSoup(res, 'html.parser') + + next_link = soup.find("a", {"rel": "next"}) + if next_link != None: + full_url = urlparse.urljoin(linksToCrawl[i], next_link['href']) + linksToCrawl.insert(i + 1, full_url) + + listOfTopics = findDescriptionPages(link) + for topic in listOfTopics: + itemPage = br.open(str(topic)) + savePage(itemPage, topic) + + except Exception as e: + print('Error getting link: ', link, e) + i += 1 + + # finalTime = time.time() + # print finalTime - initialTime + + input("Crawling Best Carding world forum done sucessfully. Press ENTER to continue\n") + + return + + +# Returns True if the link is 'Topic' Links +def isDescriptionLink(url): + if 'topic' in url: + return True + return False + + +# Returns True if the link is a listingPage link +def isListingLink(url): + ''' + reg = 'board=[0-9]+.[0-9]+\Z' + if len(re.findall(reg, url)) == 0: + return False + return True + ''' + if 'forum' in url: + return True + return False + + +# calling the parser to define the links +def findDescriptionPages(url): + soup = "" + + error = False + try: + html = codecs.open( + "C:\\Users\\Helium\\Documents\\threatIntelligence-main\\DarkWebMining_Working\\Forums\\OnniForums\\HTML_Pages\\" + str( + "%02d" % date.today().month) + str("%02d" % date.today().day) + str( + "%04d" % date.today().year) + "\\Listing\\" + getNameFromURL(url) + ".html", encoding='utf8') + soup = BeautifulSoup(html, "html.parser") + except: + try: + html = open( + "C:\\Users\\Helium\\Documents\\threatIntelligence-main\\DarkWebMining_Working\\Forums\\OnniForums\\HTML_Pages\\" + str( + "%02d" % date.today().month) + str("%02d" % date.today().day) + str( + "%04d" % date.today().year) + "\\Listing\\" + getNameFromURL(url) + ".html") + soup = BeautifulSoup(html, "html.parser") + except: + error = True + print("There was a problem to read the file " + getNameFromURL(url) + " in the listing section.") + + if not error: + return bestcardingworld_links_parser(soup) + + else: + return [] + + +def crawler(): + startCrawling() + print("Crawling and Parsing The Best Carding World .... DONE!") diff --git a/Forums/OnniForums/crawler_selenium.py b/Forums/OnniForums/crawler_selenium.py new file mode 100644 index 0000000..5839bc9 --- /dev/null +++ b/Forums/OnniForums/crawler_selenium.py @@ -0,0 +1,303 @@ +__author__ = 'DarkWeb' + +''' +OnniForums Crawler (Selenium) +''' + +from selenium import webdriver +from selenium.common.exceptions import NoSuchElementException +from selenium.webdriver.firefox.firefox_profile import FirefoxProfile +from selenium.webdriver.firefox.firefox_binary import FirefoxBinary +from selenium.webdriver.firefox.service import Service +from selenium.webdriver.common.by import By +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.support.ui import WebDriverWait +from PIL import Image + +import urllib.parse as urlparse +import os, re, time +from datetime import date +import subprocess +from bs4 import BeautifulSoup +from Forums.Initialization.prepare_parser import new_parse +from Forums.OnniForums.parser import cryptBB_links_parser +from Forums.Utilities.utilities import cleanHTML + +counter = 1 +baseURL = 'http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/' + + +# Opens Tor Browser, crawls the website +def startCrawling(): + opentor() + # forumName = getForumName() + driver = getAccess() + + if driver != 'down': + try: + login(driver) + # crawlForum(driver) + except Exception as e: + print(driver.current_url, e) + closetor(driver) + + # new_parse(forumName, False) + + +# Opens Tor Browser +def opentor(): + global pid + print("Connecting Tor...") + path = open('../../path.txt').readline().strip() + pro = subprocess.Popen(path) + pid = pro.pid + time.sleep(7.5) + input('Tor Connected. Press ENTER to continue\n') + return + + +# Login using premade account credentials and do login captcha manually +def login(driver): + #click login button + login_link = driver.find_element( + by=By.XPATH, value='/html/body/div/div[2]/div/table/tbody/tr[2]/td/center/pre/strong/a').\ + get_attribute('href') + driver.get(login_link) + + #entering username and password into input boxes + usernameBox = driver.find_element(by=By.XPATH, value='/html/body/div/div[2]/div/form/table/tbody/tr[2]/td[2]/input') + #Username here + usernameBox.send_keys('purely_cabbage') + passwordBox = driver.find_element(by=By.XPATH, value='/html/body/div/div[2]/div/form/table/tbody/tr[3]/td[2]/input') + #Password here + passwordBox.send_keys('$ourP@tchK1ds') + + input("Press ENTER when log in is completed\n") + + # wait for listing page show up (This Xpath may need to change based on different seed url) + WebDriverWait(driver, 50).until(EC.visibility_of_element_located( + (By.XPATH, '//*[@id="tab_content"]'))) + + +# Returns the name of the website +def getForumName(): + name = 'OnniForums' + return name + + +# Return the link of the website +def getFixedURL(): + url = 'http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/' + return url + + +# Closes Tor Browser +def closetor(driver): + # global pid + # os.system("taskkill /pid " + str(pro.pid)) + # os.system("taskkill /t /f /im tor.exe") + print('Closing Tor...') + driver.close() + time.sleep(3) + return + + +# Creates FireFox 'driver' and configure its 'Profile' +# to use Tor proxy and socket +def createFFDriver(): + file = open('../../path.txt', 'r') + lines = file.readlines() + + ff_binary = FirefoxBinary(lines[0].strip()) + + ff_prof = FirefoxProfile(lines[1].strip()) + ff_prof.set_preference("places.history.enabled", False) + ff_prof.set_preference("privacy.clearOnShutdown.offlineApps", True) + ff_prof.set_preference("privacy.clearOnShutdown.passwords", True) + ff_prof.set_preference("privacy.clearOnShutdown.siteSettings", True) + ff_prof.set_preference("privacy.sanitize.sanitizeOnShutdown", True) + ff_prof.set_preference("signon.rememberSignons", False) + ff_prof.set_preference("network.cookie.lifetimePolicy", 2) + ff_prof.set_preference("network.dns.disablePrefetch", True)# + ff_prof.set_preference("network.http.sendRefererHeader", 0) + ff_prof.set_preference("permissions.default.image", 3) + ff_prof.set_preference("browser.download.folderList", 2) + ff_prof.set_preference("browser.download.manager.showWhenStarting", False) + ff_prof.set_preference("browser.helperApps.neverAsk.saveToDisk", "text/plain") + ff_prof.set_preference('network.proxy.type', 1) + ff_prof.set_preference("network.proxy.socks_version", 5) + ff_prof.set_preference('network.proxy.socks', '127.0.0.1') + ff_prof.set_preference('network.proxy.socks_port', 9150) + ff_prof.set_preference('network.proxy.socks_remote_dns', True) + ff_prof.set_preference("javascript.enabled", True) + ff_prof.update_preferences() + + service = Service(lines[2].strip()) + + driver = webdriver.Firefox(firefox_binary=ff_binary, firefox_profile=ff_prof, service=service) + + return driver + + +def getAccess(): + url = getFixedURL() + driver = createFFDriver() + try: + driver.get(url) + return driver + except: + driver.close() + return 'down' + + +# Saves the crawled html page +def savePage(page, url): + cleanPage = cleanHTML(page) + filePath = getFullPathName(url) + os.makedirs(os.path.dirname(filePath), exist_ok=True) + open(filePath, 'wb').write(cleanPage.encode('utf-8')) + return + + +# Gets the full path of the page to be saved along with its appropriate file name +def getFullPathName(url): + fileName = getNameFromURL(url) + if isDescriptionLink(url): + fullPath = r'..\OnniForums\HTML_Pages\\' + str( + "%02d" % date.today().month) + str("%02d" % date.today().day) + str( + "%04d" % date.today().year) + r'\\' + r'Description\\' + fileName + '.html' + else: + fullPath = r'..\OnniForums\HTML_Pages\\' + str( + "%02d" % date.today().month) + str("%02d" % date.today().day) + str( + "%04d" % date.today().year) + r'\\' + r'Listing\\' + fileName + '.html' + return fullPath + + +# Creates the file name from passed URL +def getNameFromURL(url): + global counter + name = ''.join(e for e in url if e.isalnum()) + if (name == ''): + name = str(counter) + counter = counter + 1 + return name + + +def getInterestedLinks(): + links = [] + + # Hacking & Cracking tutorials + links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Hacking-Cracking-tutorials') + # Hacking & Cracking questions + # links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Hacking-Cracking-questions') + # Exploit PoCs + # links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Exploit-PoCs') + # Cracked software + # links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Cracked-software') + # Malware-development + # links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Malware-development') + # Carding & Fraud + # links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Carding-Fraud') + # Darknet Discussions + #links.append('http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/forumdisplay.php?fid=88') + # OPSEC + # links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-OPSEC') + # Databases + # links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Databases') + # Proxies + # links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Proxies') + + return links + + +def crawlForum(driver): + print("Crawling the OnniForums forum") + + linksToCrawl = getInterestedLinks() + visited = set(linksToCrawl) + initialTime = time.time() + + i = 0 + count = 0 + while i < len(linksToCrawl): + link = linksToCrawl[i] + print('Crawling :', link) + try: + try: + driver.get(link) + except: + driver.refresh() + html = driver.page_source + savePage(html, link) + + has_next_page = True + while has_next_page: + list = topicPages(html) + for item in list: + itemURL = urlparse.urljoin(baseURL, str(item)) + try: + driver.get(itemURL) + except: + driver.refresh() + savePage(driver.page_source, item) + driver.back() + # comment out + break + + # comment out + if count == 1: + count = 0 + break + + try: + temp = driver.find_element(by=By.XPATH, value= + '/html/body/div/div[2]/div/div[2]/div') + link = temp.find_element(by=By.CLASS_NAME, value='pagination_next').get_attribute('href') + + if link == "": + raise NoSuchElementException + try: + driver.get(link) + except: + driver.refresh() + html = driver.page_source + savePage(html, link) + count += 1 + + except NoSuchElementException: + has_next_page = False + + except Exception as e: + print(link, e) + i += 1 + + # finalTime = time.time() + # print finalTime - initialTime + + input("Crawling OnniForums forum done sucessfully. Press ENTER to continue\n") + + +# Returns 'True' if the link is Topic link +def isDescriptionLink(url): + if 'Thread' in url: + return True + return False + + +# Returns True if the link is a listingPage link +def isListingLink(url): + if 'Forum' in url: + return True + return False + + +# calling the parser to define the links +def topicPages(html): + soup = BeautifulSoup(html, "html.parser") + #print(soup.find('div', id="container").find('div', id="content").find('table', {"class": "tborder clear"}).find('tbody').find('tr',{"class": "inline_row"}).find('strong').text) + return cryptBB_links_parser(soup) + + +def crawler(): + startCrawling() + # print("Crawling and Parsing BestCardingWorld .... DONE!") diff --git a/Forums/OnniForums/geckodriver.log b/Forums/OnniForums/geckodriver.log new file mode 100644 index 0000000..d1e3763 --- /dev/null +++ b/Forums/OnniForums/geckodriver.log @@ -0,0 +1,462 @@ +1655762162153 geckodriver INFO Listening on 127.0.0.1:50333 +1655762166434 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50334" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileqUeAAN" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655762166904 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50334/devtools/browser/fb880205-35da-44a4-83b8-a861ce7125f1 +1655762168590 Marionette INFO Listening on port 50341 +1655762169104 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655762207624 Marionette INFO Stopped listening on port 50341 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function + +###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + +1655762207923 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655762990320 geckodriver INFO Listening on 127.0.0.1:50884 +1655762994595 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50885" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileurjxEe" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655762995244 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50885/devtools/browser/36503f58-7ab7-4188-81b0-fa1b81f32c0a +1655762996997 Marionette INFO Listening on port 50890 +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +1655762997277 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655763083091 Marionette INFO Stopped listening on port 50890 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1655763083216 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655763089557 geckodriver INFO Listening on 127.0.0.1:50923 +1655763093759 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50924" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile5mY1qq" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655763094208 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50924/devtools/browser/68ce2df9-521f-4258-94f7-c2e2b199701a +1655763095918 Marionette INFO Listening on port 50929 +1655763096408 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655763143935 Marionette INFO Stopped listening on port 50929 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1655763144495 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655763166975 geckodriver INFO Listening on 127.0.0.1:50947 +1655763171175 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50948" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilenAGZM5" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655763171719 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50948/devtools/browser/0b9966bd-0cec-45cc-9d11-02a2580233f0 +1655763173307 Marionette INFO Listening on port 50953 +1655763173368 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655763208922 Marionette INFO Stopped listening on port 50953 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData] +!!! error running onStopped callback: TypeError: callback is not a function +1655763209261 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655763387409 geckodriver INFO Listening on 127.0.0.1:49752 +1655763391806 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "49753" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilevAHJSJ" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655763392827 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:49753/devtools/browser/7fe49a16-940f-4ee8-9a2f-48739a92db78 +1655763394700 Marionette INFO Listening on port 49759 +1655763395103 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655763430848 Marionette INFO Stopped listening on port 49759 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData] +!!! error running onStopped callback: TypeError: callback is not a function +1655763431145 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655765589286 geckodriver INFO Listening on 127.0.0.1:50102 +1655765621677 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50103" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile8ecUpb" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655765622149 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:50103/devtools/browser/e5a916fc-987c-43d4-9c12-9c9b88cca242 +1655765623776 Marionette INFO Listening on port 50109 +1655765623892 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655766125168 Marionette INFO Stopped listening on port 50109 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1655766125303 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655766505099 geckodriver INFO Listening on 127.0.0.1:50156 +1655766509351 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50157" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileVqvmzh" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655766509876 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50157/devtools/browser/13084c14-64d3-48a5-8b99-4c514e961d4f +1655766511328 Marionette INFO Listening on port 50162 +1655766511395 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655766552298 Marionette INFO Stopped listening on port 50162 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData] +!!! error running onStopped callback: TypeError: callback is not a function +1655766552578 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655767657126 geckodriver INFO Listening on 127.0.0.1:50257 +1655767661375 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50258" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileB9Dzeh" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655767661843 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50258/devtools/browser/867cd318-6430-4b3f-9b63-8c6bdf17636a +1655767663330 Marionette INFO Listening on port 50263 +1655767663431 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655767668263 Marionette INFO Stopped listening on port 50263 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function + +###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + +1655767668591 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655767689480 geckodriver INFO Listening on 127.0.0.1:50276 +1655767693635 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50277" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile8ZOQe7" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655767694215 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50277/devtools/browser/a70a3f5b-19af-4a68-99c8-c46086ba9599 +1655767695725 Marionette INFO Listening on port 50282 +1655767695830 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655767701272 Marionette INFO Stopped listening on port 50282 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1655767701594 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655767791360 geckodriver INFO Listening on 127.0.0.1:50296 +1655767795575 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50297" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile4GEqBr" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655767796001 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50297/devtools/browser/e2d4e57a-16ce-429d-9804-e9e391199ddf +1655767797534 Marionette INFO Listening on port 50302 +1655767797661 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655767802833 Marionette INFO Stopped listening on port 50302 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1655767803151 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655767902908 geckodriver INFO Listening on 127.0.0.1:50320 +1655767907009 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50321" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileqlJ8ZA" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655767907484 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:50321/devtools/browser/49e77669-01dd-447e-a804-36d42b3400cf +1655767908782 Marionette INFO Listening on port 50326 +1655767909069 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655767916218 Marionette INFO Stopped listening on port 50326 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1655767916523 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655767958348 geckodriver INFO Listening on 127.0.0.1:50340 +1655767962557 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50341" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilekpCdfs" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655767963032 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50341/devtools/browser/b4d1728c-d26f-4537-adc2-5c954b171b13 +1655767964591 Marionette INFO Listening on port 50346 +1655767964633 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655767973210 Marionette INFO Stopped listening on port 50346 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1655767973501 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655767998084 geckodriver INFO Listening on 127.0.0.1:50358 +1655768002314 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50359" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileujYNBj" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655768002757 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50359/devtools/browser/04112c45-204d-43f1-9615-34782fd06632 +1655768004165 Marionette INFO Listening on port 50364 +1655768004375 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655768009787 Marionette INFO Stopped listening on port 50364 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] + +###!!! [Child][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + +!!! error running onStopped callback: TypeError: callback is not a function + +###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + +1655768010100 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655768083966 geckodriver INFO Listening on 127.0.0.1:50386 +1655768088234 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50387" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile39Pl1f" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655768088664 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50387/devtools/browser/3d6cda42-cb60-4c67-8d76-1d186a51887f +1655768090263 Marionette INFO Listening on port 50392 +1655768090299 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655768091655 Marionette INFO Stopped listening on port 50392 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function + +###!!! [Parent][MessageChannel] Error: (msgtype=0x390076,name=PContent::Msg_DestroyBrowsingContextGroup) Closed channel: cannot send/recv + + +###!!! [Parent][MessageChannel] Error: (msgtype=0x390076,name=PContent::Msg_DestroyBrowsingContextGroup) Closed channel: cannot send/recv + + +###!!! [Parent][MessageChannel] Error: (msgtype=0x390076,name=PContent::Msg_DestroyBrowsingContextGroup) Closed channel: cannot send/recv + +1655768092659 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655768101201 geckodriver INFO Listening on 127.0.0.1:50404 +1655768105373 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50405" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile2YRGSz" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655768105853 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50405/devtools/browser/9c4a4d71-25ff-4980-a54d-8545d6200790 +1655768107431 Marionette INFO Listening on port 50410 +1655768107530 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655768119155 Marionette INFO Stopped listening on port 50410 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1655768119438 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655768229472 geckodriver INFO Listening on 127.0.0.1:50431 +1655768233458 webdriver::server WARN Rejected request with missing Host header diff --git a/Forums/OnniForums/parser.py b/Forums/OnniForums/parser.py new file mode 100644 index 0000000..b6772f9 --- /dev/null +++ b/Forums/OnniForums/parser.py @@ -0,0 +1,354 @@ +__author__ = 'DarkWeb' + +# Here, we are importing the auxiliary functions to clean or convert data +from Forums.Utilities.utilities import * +from datetime import date +from datetime import timedelta +import re + +# Here, we are importing BeautifulSoup to search through the HTML tree +from bs4 import BeautifulSoup + +# This is the method to parse the Description Pages (one page to each topic in the Listing Pages) + +def cryptBB_description_parser(soup): + + # Fields to be parsed + + topic = "-1" # topic name + user = [] # all users of each post + addDate = [] # all dated of each post + feedback = [] # all feedbacks of each vendor (this was found in just one Forum and with a number format) + status = [] # all user's authority in each post such as (adm, member, dangerous) + reputation = [] # all users's karma in each post (usually found as a number) + sign = [] # all user's signature in each post (usually a standard message after the content of the post) + post = [] # all messages of each post + interest = [] # all user's interest in each post + + # Finding the topic (should be just one coming from the Listing Page) + + li = soup.find("td", {"class": "thead"}).find('strong') + topic = li.text + topic = re.sub("\[\w*\]", '', topic) + + topic = topic.replace(",","") + topic = topic.replace("\n","") + topic = cleanString(topic.strip()) + print(topic) + # Finding the repeated tag that corresponds to the listing of posts + + # posts = soup.find("form", {"name": "quickModForm"}).findAll('div', {"class": "windowbg"}) + \ + # soup.find("form", {"name": "quickModForm"}).findAll('div', {"class": "windowbg2"}) + + try: + posts = soup.find('table', {"class": "tborder tfixed clear"}).find('td', {"id": "posts_container"}).find_all( + 'div', {"class": "post"}) + # print(len(posts)) + + # For each message (post), get all the fields we are interested to: + + for ipost in posts: + + # Finding a first level of the HTML page + + # post_wrapper = ipost.find('div', {"class": "post_wrapper"}).find('div', {"class": "poster"}) + post_wrapper = ipost.find('span', {"class": "largetext"}) + # Finding the author (user) of the post + + # author = post_wrapper.find('h4') + author = post_wrapper.text.strip() + # print("author " + author) + user.append(cleanString(author)) # Remember to clean the problematic characters + + # Finding the status of the author + + smalltext = ipost.find('div', {"class": "post_author"}) + + # Testing here two possibilities to find this status and combine them + if ipost.find('div', {"class": "deleted_post_author"}): + status.append(-1) + interest.append(-1) + reputation.append(-1) + addDate.append(-1) + post.append("THIS POST HAS BEEN REMOVED!") + sign.append(-1) + feedback.append(-1) + continue + + # CryptBB does have membergroup and postgroup + + membergroup = smalltext.find('div', {"class": "profile-rank"}) + postgroup = smalltext.find('div', {"class": "postgroup"}) + if membergroup != None: + membergroup = membergroup.text.strip() + if postgroup != None: + postgroup = postgroup.text.strip() + membergroup = membergroup + " - " + postgroup + else: + if postgroup != None: + membergroup = postgroup.text.strip() + else: + membergroup = "-1" + + status.append(cleanString(membergroup)) + # print("status " + cleanString(membergroup)) + # Finding the interest of the author + # CryptBB does not have blurb + blurb = smalltext.find('li', {"class": "blurb"}) + if blurb != None: + blurb = blurb.text.strip() + else: + blurb = "-1" + interest.append(cleanString(blurb)) + + # Finding the reputation of the user + # CryptBB does have reputation + author_stats = smalltext.find('div', {"class": "author_statistics"}) + karma = author_stats.find('strong') + if karma != None: + karma = karma.text + karma = karma.replace("Community Rating: ", "") + karma = karma.replace("Karma: ", "") + karma = karma.strip() + else: + karma = "-1" + reputation.append(cleanString(karma)) + # print("karma " + cleanString(karma)) + # Getting here another good tag to find the post date, post content and users' signature + + postarea = ipost.find('div', {"class": "post_content"}) + + dt = postarea.find('span', {"class": "post_date"}).text + # dt = dt.strip().split() + dt = dt.strip() + day=date.today() + if "Yesterday" in dt: + yesterday = day - timedelta(days=1) + yesterday = yesterday.strftime('%m-%d-%Y') + stime = dt.replace('Yesterday,','').strip() + date_time_obj = yesterday+ ', '+stime + date_time_obj = datetime.strptime(date_time_obj,'%m-%d-%Y, %I:%M %p') + elif "hours ago" in dt: + day = day.strftime('%m-%d-%Y') + date_time_obj = postarea.find('span', {"class": "post_date"}).find('span')['title'] + date_time_obj = datetime.strptime(date_time_obj, '%m-%d-%Y, %I:%M %p') + else: + date_time_obj = datetime.strptime(dt, '%m-%d-%Y, %I:%M %p') + stime = date_time_obj.strftime('%b %d, %Y') + sdate = date_time_obj.strftime('%I:%M %p') + + + addDate.append(date_time_obj) + # print("date " + str(date_time_obj)) + # Finding the date of the post + # date_time_obj = datetime.strptime(dt, '%a %b %d, %Y %I:%M %p') + # smalltext = postarea.find('div', {"class": "flow_hidden"}).find('div', {"class": "keyinfo"})\ + # .find('div', {"class": "smalltext"}) + # sdatetime = smalltext.text + # sdatetime = sdatetime.replace(u"\xab","") # Removing unnecessary characters + # sdatetime = sdatetime.replace(u"\xbb","") # Removing unnecessary characters + # sdatetime = sdatetime.split("on: ") # Removing unnecessary characters + # sdatetime = sdatetime[1].strip() + # stime = sdatetime[:-12:-1] # Finding the time of the post + # stime = stime[::-1] + # sdate = sdatetime.replace(stime,"") # Finding the date of the post + # sdate = sdate.replace(",","") + # sdate = sdate.strip() + + # Covert the date of the post that can be informed as: "12 February 2016", "today", "yesterday". We need + # a date format here as "mm/dd/yyyy" + + # addDate.append(convertDate(sdate,"english", crawlerDate) + " " + stime) + + # Finding the post + + inner = postarea.find('div', {"class": "post_body scaleimages"}) + inner = inner.text.strip() + # print(inner) + post.append(cleanString(inner)) + + # Finding the users's signature + + # signature = ipost.find('div', {"class": "post_wrapper"}).find('div', {"class": "moderatorbar"}).find('div', {"class": "signature"}) + signature = ipost.find('div', {"class": "signature scaleimages"}) + if signature != None: + signature = signature.text.strip() + # print(signature) + else: + signature = "-1" + sign.append(cleanString(signature)) + + # As no information about users's feedback was found, just assign "-1" to the variable + + feedback.append("-1") + except: + if soup.find('td', {"class": "trow1"}).text == " You do not have permission to access this page. ": + user.append("-1") + status.append(-1) + interest.append(-1) + reputation.append(-1) + addDate.append(-1) + post.append("NO ACCESS TO THIS PAGE!") + sign.append(-1) + feedback.append(-1) + + + # Populate the final variable (this should be a list with all fields scraped) + + row = (topic, post, user, addDate, feedback, status, reputation, sign, interest) + + # Sending the results + + return row + +# This is the method to parse the Listing Pages (one page with many posts) + +def cryptBB_listing_parser(soup): + + board = "-1" # board name (the previous level of the topic in the Forum categorization tree. + # For instance: Security/Malware/Tools to hack Facebook. The board here should be Malware) + + nm = 0 # this variable should receive the number of topics + topic = [] # all topics + user = [] # all users of each topic + post = [] # number of posts of each topic + view = [] # number of views of each topic + addDate = [] # when the topic was created (difficult to find) + href = [] # this variable should receive all cleaned urls (we will use this to do the marge between + # Listing and Description pages) + + # Finding the board (should be just one) + + board = soup.find('span', {"class": "active"}).text + board = cleanString(board.strip()) + + # Finding the repeated tag that corresponds to the listing of topics + + itopics = soup.find_all('tr',{"class": "inline_row"}) + index = 0 + for itopic in itopics: + + # For each topic found, the structure to get the rest of the information can be of two types. Testing all of them + # to don't miss any topic + + + # Adding the topic to the topic list + try: + topics = itopic.find('span', {"class": "subject_old"}).find('a').text + except: + topics = itopic.find('span', {"class": "subject_new"}).find('a').text + topics = re.sub("\[\w*\]", '', topics) + topic.append(cleanString(topics)) + + # Counting how many topics we have found so far + + nm = len(topic) + + # Adding the url to the list of urls + try: + link = itopic.find('span', {"class": "subject_old"}).find('a').get('href') + except: + link = itopic.find('span',{"class": "subject_new"}).find('a').get('href') + link = cleanLink(link) + href.append(link) + + # Finding the author of the topic + ps = itopic.find('div', {"class":"author smalltext"}).find('a').text + author = ps.strip() + user.append(cleanString(author)) + + # Finding the number of replies + columns = itopic.findChildren('td',recursive=False) + posts = columns[3].text + + post.append(cleanString(posts)) + + # Finding the number of Views + tview = columns[4].text + view.append(cleanString(tview)) + + + + # If no information about when the topic was added, just assign "-1" to the variable + #dt = itopic.find('div', {"class": "responsive-hide"}).text.split('ยป')[1] + #dt = dt.strip() + #date_time_obj = datetime.strptime(dt,'%a %b %d, %Y %I:%M %p') + #addDate.append(date_time_obj) + addDate.append("-1") + + + + index += 1 + return organizeTopics("CryptBB", nm, topic, board, view, post, user, addDate, href) + + # if len(tag) > 0: + # + # # Finding the topic + # + # tds = tds[0].find(tag[0]) + # topics = tds.text + # topics = topics.replace(u"\xbb","") + # topics = topics.strip() + # topic.append(cleanString(topics)) + # + # # Counting how many topics we have found so far + # + # nm = len(topic) + # + # # Adding the url to the list of urls + # + # link = tds.findAll('a', href=True) + # link = link[0].get('href') + # link = cleanLink(link) + # href.append(link) + # + # # Finding the author of the topic + # + # ps = itopic.find('td', {"class": tag[1]}).find('p').find('a') + # if ps == None: + # ps = itopic.find('td', {"class": tag[1]}).find('p') + # ps = ps.text.replace("Started by ","") + # else: + # ps = ps.text + # author = ps.strip() + # user.append(cleanString(author)) + # + # # Finding the number of replies + # + # statistics = itopic.find('td', {"class": tag[2]}) + # statistics = statistics.text + # statistics = statistics.split("Replies") + # posts = statistics[0].strip() + # post.append(cleanString(posts)) + # + # # Finding the number of Views + # + # views = statistics[1] + # views = views.replace("Views","") + # views = views.strip() + # view.append(cleanString(views)) + # + # # As no information about when the topic was added, just assign "-1" to the variable + # + # addDate.append("-1") + + #return organizeTopics("TheMajesticGarden", nm, topic, board, view, post, user, addDate, href) + +def cryptBB_links_parser(soup): + + # Returning all links that should be visited by the Crawler + + href = [] + #print(soup.find('table', {"class": "tborder clear"}).find( + # 'tbody').find_all('tr', {"class": "inline_row"})) + listing = soup.find('table', {"class": "tborder clear"}).find('tbody').find_all('tr', {"class": "inline_row"}) + + for a in listing: + try: + link = a.find('span', {"class": "subject_old"}).find('a').get('href') + except: + link = a.find('span', {"class": "subject_new"}).find('a').get('href') + + href.append(link) + + return href \ No newline at end of file diff --git a/path.txt b/path.txt index 743d2d1..3992963 100644 --- a/path.txt +++ b/path.txt @@ -1,3 +1,3 @@ -C:\Users\calsyslab\Desktop\Tor Browser\Browser\firefox.exe -C:\Users\calsyslab\Desktop\Tor Browser\Browser\TorBrowser\Data\Browser\profile.default -C:\Users\calsyslab\Documents\CALSysLab\threatIntelligence\DarkWebMining_Working\selenium\geckodriver.exe \ No newline at end of file +C:\Users\Helium\Desktop\Tor Browser\Browser\firefox.exe +C:\Users\Helium\Desktop\Tor Browser\Browser\TorBrowser\Data\Browser\profile.default +C:\Users\Helium\PycharmProjects\dw_pipeline_test\selenium\geckodriver.exe \ No newline at end of file