m00nkey scraper completed

MarketPlaces/DarkMatter/parser.py

@@ -11,7 +11,7 @@ from bs4 import BeautifulSoup
 #stores info it needs in different lists, these lists are returned after being organized
 #@param: soup object looking at html page of description page
 #return: 'row' that contains a variety of lists that each hold info on the description page
-def darkfox_description_parser(soup):
+def darkmatter_description_parser(soup):
 
     # Fields to be parsed
 
@@ -84,11 +84,11 @@ def darkfox_description_parser(soup):
 #stores info it needs in different lists, these lists are returned after being organized
 #@param: soup object looking at html page of listing page
 #return: 'row' that contains a variety of lists that each hold info on the listing page
-def darkfox_listing_parser(soup):
+def darkmatter_listing_parser(soup):
 
     # Fields to be parsed
     nm = 0                                    # Total_Products (Should be Integer)
-    mktName = "DarkFox"                       # 0 Marketplace_Name
+    mktName = "DarkMatter"                       # 0 Marketplace_Name
     name = []                                 # 1 Product_Name
     CVE = []                                  # 2 Product_CVE_Classification (Common Vulnerabilities and Exposures)
     MS = []                                   # 3 Product_MS_Classification (Microsoft Security)

MarketPlaces/Initialization/geckodriver.log

@@ -15930,3 +15930,220 @@ unwatchForTargets()@TargetList.jsm:37
 destructor()@TargetList.jsm:109
 stop()@CDP.jsm:104
 close()@RemoteAgent.jsm:138
+1689712165507	geckodriver	INFO	Listening on 127.0.0.1:52569
+1689712171396	mozrunner::runner	INFO	Running command: "C:\\Users\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "52570" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofileBYUxGv"
+console.log: "TorSettings: loadFromPrefs()"
+console.log: "TorConnect: init()"
+console.log: "TorConnect: Entering Initial state"
+console.log: "TorConnect: Observed profile-after-change"
+console.log: "TorConnect: Observing topic 'TorProcessExited'"
+console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'"
+console.log: "TorConnect: Observing topic 'torsettings:ready'"
+console.log: "TorSettings: Observed profile-after-change"
+1689712173264	Marionette	INFO	Marionette enabled
+console.log: "TorConnect: Will load after bootstrap => [about:blank]"
+console.error: "Could not load engine [email protected]: Error: Extension is invalid"
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
+DevTools listening on ws://localhost:52570/devtools/browser/46c17946-d8da-4f79-89da-b48961358c5e
+1689712177652	Marionette	INFO	Listening on port 52577
+1689712178244	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+1689712395837	Marionette	INFO	Stopped listening on port 52577
+JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
+JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData]
+!!! error running onStopped callback: TypeError: callback is not a function
+JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first.
+JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofileBYUxGv\thumbnails) because it does not exist
+[Parent 6172, IPC I/O Parent] WARNING: file /var/tmp/build/firefox-b6010b1466c9/ipc/chromium/src/base/process_util_win.cc:167
+1689712396253	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
+unwatchForTabs()@TargetList.jsm:70
+unwatchForTargets()@TargetList.jsm:37
+destructor()@TargetList.jsm:109
+stop()@CDP.jsm:104
+close()@RemoteAgent.jsm:138
+1689712458291	geckodriver	INFO	Listening on 127.0.0.1:52745
+1689712464089	mozrunner::runner	INFO	Running command: "C:\\Users\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "52746" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofile7pa8h1"
+console.log: "TorSettings: loadFromPrefs()"
+console.log: "TorConnect: init()"
+console.log: "TorConnect: Entering Initial state"
+console.log: "TorConnect: Observed profile-after-change"
+console.log: "TorConnect: Observing topic 'TorProcessExited'"
+console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'"
+console.log: "TorConnect: Observing topic 'torsettings:ready'"
+console.log: "TorSettings: Observed profile-after-change"
+1689712465452	Marionette	INFO	Marionette enabled
+console.log: "TorConnect: Will load after bootstrap => [about:blank]"
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+console.error: "Could not load engine [email protected]: Error: Extension is invalid"
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
+DevTools listening on ws://localhost:52746/devtools/browser/f1aa2cdb-25e1-4240-992b-5dc17ec4b35a
+1689712468928	Marionette	INFO	Listening on port 52754
+1689712469432	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+JavaScript error: resource://gre/actors/AutoCompleteChild.jsm, line 125: Error: Invalid autocomplete selectedIndex
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+1689712803150	Marionette	INFO	Stopped listening on port 52754
+JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
+!!! error running onStopped callback: TypeError: callback is not a function
+JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first.
+JavaScript error: resource://gre/modules/PageThumbs.jsm, line 709: AbortError: IOUtils.profileBeforeChange getter: IOUtils: profileBeforeChange phase has already finished
+
+###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost
+
+1689712803626	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
+unwatchForTabs()@TargetList.jsm:70
+unwatchForTargets()@TargetList.jsm:37
+destructor()@TargetList.jsm:109
+stop()@CDP.jsm:104
+close()@RemoteAgent.jsm:138
+1689725884014	geckodriver	INFO	Listening on 127.0.0.1:57758
+1689725890715	mozrunner::runner	INFO	Running command: "C:\\Users\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "57759" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofileKZtDvb"
+console.log: "TorSettings: loadFromPrefs()"
+console.log: "TorConnect: init()"
+console.log: "TorConnect: Entering Initial state"
+console.log: "TorConnect: Observed profile-after-change"
+console.log: "TorConnect: Observing topic 'TorProcessExited'"
+console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'"
+console.log: "TorConnect: Observing topic 'torsettings:ready'"
+console.log: "TorSettings: Observed profile-after-change"
+1689725894724	Marionette	INFO	Marionette enabled
+console.log: "TorConnect: Will load after bootstrap => [about:blank]"
+JavaScript error: resource:///modules/UrlbarInput.jsm, line 2952: TypeError: event.target.closest is not a function
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+console.error: "Could not load engine [email protected]: Error: Extension is invalid"
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
+DevTools listening on ws://localhost:57759/devtools/browser/5130c45f-e57f-4719-970e-f1ca80613eda
+1689725901218	Marionette	INFO	Listening on port 57767
+1689725902070	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+
+###!!! [Child][MessageChannel] Error: (msgtype=0x390097,name=PContent::Msg_InitBackground) Channel closing: too late to send/recv, messages will be lost
+
+1689725909826	Marionette	INFO	Stopped listening on port 57767
+JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
+JavaScript error: resource://gre/modules/AsyncShutdown.jsm, line 575: AbortError: Actor 'Conduits' destroyed before query 'RunListener' was resolved
+!!! error running onStopped callback: TypeError: callback is not a function
+JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first.
+JavaScript error: resource://gre/modules/PageThumbs.jsm, line 709: AbortError: IOUtils.profileBeforeChange getter: IOUtils: profileBeforeChange phase has already finished
+[Parent 4420, IPC I/O Parent] WARNING: file /var/tmp/build/firefox-b6010b1466c9/ipc/chromium/src/base/process_util_win.cc:167
+1689725911309	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
+unwatchForTabs()@TargetList.jsm:70
+unwatchForTargets()@TargetList.jsm:37
+destructor()@TargetList.jsm:109
+stop()@CDP.jsm:104
+close()@RemoteAgent.jsm:138
+1689726298926	geckodriver	INFO	Listening on 127.0.0.1:57942
+1689726305572	mozrunner::runner	INFO	Running command: "C:\\Users\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "57943" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofile4REPVd"
+console.log: "TorSettings: loadFromPrefs()"
+console.log: "TorConnect: init()"
+console.log: "TorConnect: Entering Initial state"
+console.log: "TorConnect: Observed profile-after-change"
+console.log: "TorConnect: Observing topic 'TorProcessExited'"
+console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'"
+console.log: "TorConnect: Observing topic 'torsettings:ready'"
+console.log: "TorSettings: Observed profile-after-change"
+1689726307885	Marionette	INFO	Marionette enabled
+console.log: "TorConnect: Will load after bootstrap => [about:blank]"
+console.error: "Could not load engine [email protected]: Error: Extension is invalid"
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
+DevTools listening on ws://localhost:57943/devtools/browser/77548608-482e-4f84-83b5-ea9ed5de1e6c
+1689726313342	Marionette	INFO	Listening on port 57950
+1689726313710	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+1689726526374	Marionette	INFO	Stopped listening on port 57950
+JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
+!!! error running onStopped callback: TypeError: callback is not a function
+JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first.
+JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofile4REPVd\thumbnails) because it does not exist
+[Parent 6328, IPC I/O Parent] WARNING: file /var/tmp/build/firefox-b6010b1466c9/ipc/chromium/src/base/process_util_win.cc:167
+[Parent 6328, IPC I/O Parent] WARNING: pipe error: 232: file /var/tmp/build/firefox-b6010b1466c9/ipc/chromium/src/chrome/common/ipc_channel_win.cc:544
+1689726527913	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
+unwatchForTabs()@TargetList.jsm:70
+unwatchForTargets()@TargetList.jsm:37
+destructor()@TargetList.jsm:109
+stop()@CDP.jsm:104
+close()@RemoteAgent.jsm:138
+1689727835682	geckodriver	INFO	Listening on 127.0.0.1:58463
+1689727842340	mozrunner::runner	INFO	Running command: "C:\\Users\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "58464" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofileSzsJi7"
+console.log: "TorSettings: loadFromPrefs()"
+console.log: "TorConnect: init()"
+console.log: "TorConnect: Entering Initial state"
+console.log: "TorConnect: Observed profile-after-change"
+console.log: "TorConnect: Observing topic 'TorProcessExited'"
+console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'"
+console.log: "TorConnect: Observing topic 'torsettings:ready'"
+console.log: "TorSettings: Observed profile-after-change"
+1689727844481	Marionette	INFO	Marionette enabled
+console.log: "TorConnect: Will load after bootstrap => [about:blank]"
+console.error: "Could not load engine [email protected]: Error: Extension is invalid"
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
+DevTools listening on ws://localhost:58464/devtools/browser/d34aa7cb-51f8-41fe-9e31-7a0ac5675ef9
+1689727850376	Marionette	INFO	Listening on port 58471
+1689727851163	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+JavaScript error: resource://gre/actors/AutoCompleteChild.jsm, line 125: Error: Invalid autocomplete selectedIndex
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/b0ecb48a-4f1b-4898-8d6c-86f25b19102b'"  nsresult: "0x805303f4 (<unknown>)"  location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670"  data: yes]
+Stack: mainThreadFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15
+mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10
+_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22
+urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17
+_resurrectSource@resource://devtools/server/actors/thread.js:2142:35
+addAllSources@resource://devtools/server/actors/thread.js:1509:14
+watch@resource://devtools/server/actors/resources/sources.js:52:17
+watchResources@resource://devtools/server/actors/resources/index.js:239:19
+_watchTargetResources@resource://devtools/server/actors/targets/target-actor-mixin.js:156:24
+addWatcherDataEntry@resource://devtools/server/actors/targets/target-actor-mixin.js:47:20
+_addWatcherDataEntry@resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm:483:24
+receiveMessage@resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm:425:21
+Line: 670, column: 0
+console.error: ({})
+SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/37f88ac5-4cdf-4bdc-b792-480b4f61196c'"  nsresult: "0x805303f4 (<unknown>)"  location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670"  data: yes]
+Stack: mainThreadFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15
+mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10
+_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22
+urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17
+_resurrectSource@resource://devtools/server/actors/thread.js:2142:35
+addAllSources@resource://devtools/server/actors/thread.js:1509:14
+watch@resource://devtools/server/actors/resources/sources.js:52:17
+watchResources@resource://devtools/server/actors/resources/index.js:239:19
+_watchTargetResources@resource://devtools/server/actors/targets/target-actor-mixin.js:156:24
+addWatcherDataEntry@resource://devtools/server/actors/targets/target-actor-mixin.js:47:20
+_addWatcherDataEntry@resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm:483:24
+receiveMessage@resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm:425:21
+Line: 670, column: 0
+console.error: ({})
+JavaScript error: resource://devtools/shared/DevToolsUtils.js, line 670: Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/b0ecb48a-4f1b-4898-8d6c-86f25b19102b'
+JavaScript error: resource://devtools/shared/DevToolsUtils.js, line 670: Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/37f88ac5-4cdf-4bdc-b792-480b4f61196c'
+1689728550247	Marionette	INFO	Stopped listening on port 58471
+JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
+JavaScript error: resource:///actors/RFPHelperParent.jsm, line 28: TypeError: browser is null
+JavaScript error: resource:///actors/RFPHelperParent.jsm, line 28: TypeError: browser is null
+JavaScript error: resource:///actors/RFPHelperParent.jsm, line 28: TypeError: browser is null
+!!! error running onStopped callback: TypeError: callback is not a function
+JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first.
+JavaScript error: resource://gre/modules/PageThumbs.jsm, line 709: AbortError: IOUtils.profileBeforeChange getter: IOUtils: profileBeforeChange phase has already finished
+[Parent 5584, IPC I/O Parent] WARNING: file /var/tmp/build/firefox-b6010b1466c9/ipc/chromium/src/base/process_util_win.cc:167
+1689728551276	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
+unwatchForTabs()@TargetList.jsm:70
+unwatchForTargets()@TargetList.jsm:37
+destructor()@TargetList.jsm:109
+stop()@CDP.jsm:104
+close()@RemoteAgent.jsm:138

MarketPlaces/M00nkeyMarket/crawler_selenium.py

@@ -33,16 +33,16 @@ MARKET_NAME = 'M00nkeyMarket'
 # Opens Tor Browser, crawls the website, then parses, then closes tor
 #acts like the main method for the crawler, another function at the end of this code calls this function later
 def startCrawling():
-    # opentor()
+    opentor()
     driver = getAccess()
-    
-    # if driver != 'down':
-    #     try:
-    #         login(driver)
-    #         crawlForum(driver)
-    #     except Exception as e:
-    #         print(driver.current_url, e)
-    #     closetor(driver)
+
+    if driver != 'down':
+        try:
+            login(driver)
+            crawlForum(driver)
+        except Exception as e:
+            print(driver.current_url, e)
+        closetor(driver)
 
     new_parse(MARKET_NAME, BASE_URL, False)
 
@@ -203,7 +203,7 @@ def getInterestedLinks():
     # software
     links.append('http://moonkey4f2mkcp6hpackeea356puiry27h3dz3hzbt3adbmsk4gs7wyd.onion/search/subcategories?subcategory=30')
     # # guides
-    links.append('http://moonkey4f2mkcp6hpackeea356puiry27h3dz3hzbt3adbmsk4gs7wyd.onion/search/subcategories?subcategory=17')
+    # links.append('http://moonkey4f2mkcp6hpackeea356puiry27h3dz3hzbt3adbmsk4gs7wyd.onion/search/subcategories?subcategory=17')
 
     return links
 
@@ -243,11 +243,11 @@ def crawlForum(driver):
                     driver.back()
 
                     # comment out
-                    break
+                    # break
 
                 # comment out
-                if count == 1:
-                    break
+                # if count == 1:
+                #     break
 
                 try:
                     link = driver.find_element(by=By.LINK_TEXT, value='Next ›').get_attribute('href')

MarketPlaces/M00nkeyMarket/parser.py

@@ -35,22 +35,30 @@ def m00nkey_description_parser(soup):
     shipTo = "-1"  # 18 Product_ShippedTo
 
     #vendor name
-    temp = soup.find('div', {'class': 'box rounded mb-0'}).find('a').text
-    vendor = (cleanString(temp.strip()))
+    try:
+        temp = soup.find('div', {'class': 'box rounded mb-0'}).find('a').text
+        vendor = (cleanString(temp.strip()))
+    except:
+        vendor = "-1"
 
     #successful transaction
-    temp = soup.findAll('div', {'class','text-center text-truncate column-flex ml-1 mr-1'}) #card sidebar-menu mb-4 card sidebar-menu mb-4
-    temp2 = temp[1].findAll('span', {'class', 'float-right font-weight-bold'})
-    temp = temp2[1].text
-    success = (temp.strip())
-    # print(success)
+    try:
+        temp = soup.findAll('div', {'class','text-center text-truncate column-flex ml-1 mr-1'}) #card sidebar-menu mb-4 card sidebar-menu mb-4
+        temp2 = temp[1].findAll('span', {'class', 'float-right font-weight-bold'})
+        temp = temp2[1].text
+        success = (temp.strip())
+    except:
+        print("success")
 
 
     #vendor rating 5
-    temp = soup.findAll('div', {'class', 'text-center text-truncate column-flex ml-1 mr-1'})  # card sidebar-menu mb-4 card sidebar-menu mb-4
-    temp2 = temp[1].findAll('span', {'class', 'float-right font-weight-bold'})
-    temp = temp2[5].text
-    rating_vendor = (cleanString(temp.strip()))
+    try:
+        temp = soup.findAll('div', {'class', 'text-center text-truncate column-flex ml-1 mr-1'})  # card sidebar-menu mb-4 card sidebar-menu mb-4
+        temp2 = temp[1].findAll('span', {'class', 'float-right font-weight-bold'})
+        temp = temp2[5].text
+        rating_vendor = (cleanString(temp.strip()))
+    except:
+        print("product")
 
     # product name
     try:
@@ -90,35 +98,58 @@ def m00nkey_description_parser(soup):
         views = cleanString((temp.strip()))
     except:
         print('Product number of view')
-        # views = "-1"
 
     reviews = "-1"  # 9 Product_Number_Of_Reviews
     rating_item = "-1"  # 10 Product_Rating
     addDate = "-1"  # 11 Product_AddedDate
 
     #BTC selling price box box-rounded mt-2
-    temp = soup.find('div', {'class', 'box box-rounded mt-2'})
-    temp2 = temp.findAll('i', {'class', 'float-right color-prices'})
-    temp = temp2[1].text
-    BTC = cleanString((temp.strip()))
+    try:
+        temp = soup.find('div', {'class', 'box box-rounded mt-2'})
+        temp2 = temp.findAll('i', {'class', 'float-right color-prices'})
+        temp = temp2[1].text
+        BTC = cleanString((temp.strip()))
+    except:
+        try:
+            temp = soup.find('div', {'class', 'box box-rounded'})
+            temp2 = temp.findAll('span', {'class', 'float-right color-prices'})
+            temp = temp2[1].text
+            BTC = cleanString((temp.strip()))
+        except:
+            print("BTC")
+
 
     # USD selling price
-    temp = soup.find('div', {'class', 'box box-rounded mt-2'})
-    temp2 = temp.findAll('center')
-    temp = temp2[1].find('i').text
-    if "$" in temp:
-        temp = temp.replace("$", "")
-    USD = cleanString((temp.strip()))
+    try:
+        temp = soup.find('div', {'class', 'box box-rounded mt-2'})
+        temp2 = temp.findAll('center')
+        temp = temp2[1].find('i').text
+        if "$" in temp:
+            temp = temp.replace("$", "")
+        USD = cleanString((temp.strip()))
+    except:
+        try:
+            temp = soup.find('div', {'class', 'box box-rounded'})
+            temp2 = temp.findAll('center')
+            temp = temp2[1].find('span').text
+            if "$" in temp:
+                temp = temp.replace("$", "")
+            USD = cleanString((temp.strip()))
+        except:
+            print("USD")
 
     EURO = "-1"  # 14 Product_EURO_SellingPrice
 
 
    # product sold
-    temp = soup.find('div', {'class', 'box rounded mb-0'})  # card sidebar-menu mb-4 card sidebar-menu mb-4
-    temp2 = temp.find('i')
-    temp = temp2.text
-    sold = (cleanString(temp.strip()))
-    # sold = "-1"
+    try:
+        temp = soup.find('div', {'class', 'box rounded mb-0'})  # card sidebar-menu mb-4 card sidebar-menu mb-4
+        temp2 = temp.find('i')
+        temp = temp2.text
+        sold = (cleanString(temp.strip()))
+        # sold = "-1"
+    except:
+        print("product sold")
 
     # product quantatiy left  ###ERRROR
     try: