Merge branch 'main' into 'josh-development'

# Conflicts:
#   setup.ini

MarketPlaces/DB_Connection/db_connection.py

@@ -139,7 +139,6 @@ def create_vendor(cur, row, marketId):
 
 
 def create_items(cur, row, marketId, vendorId):
-    print(row)
 
     sql = "Insert into items (market_id, vendor_id, name_item, description_item, cve_item, ms_item, category_item, " \
           "views_item, reviews_item, rating_item, dateadded_item, btc_item, usd_item, euro_item, quantitysold_item, " \

MarketPlaces/DarkMatter/parser.py

@@ -11,133 +11,70 @@ from bs4 import BeautifulSoup
 #stores info it needs in different lists, these lists are returned after being organized
 #@param: soup object looking at html page of description page
 #return: 'row' that contains a variety of lists that each hold info on the description page
-def darkfox_description_parser(soup):
+def darkmatter_description_parser(soup):
 
     # Fields to be parsed
 
-    name = "-1"                         # 0 Product_Name
-    describe = "-1"                     # 1 Product_Description
-    lastSeen = "-1"                     # 2 Product_LastViewDate
-    rules = "-1"                        # 3 NOT USED ...
-    CVE = "-1"                          # 4 Product_CVE_Classification (Common Vulnerabilities and Exposures)
-    MS = "-1"                           # 5 Product_MS_Classification (Microsoft Security)
-    review = "-1"                       # 6 Product_Number_Of_Reviews
-    category = "-1"                     # 7 Product_Category
-    shipFrom = "-1"                     # 8 Product_ShippedFrom
-    shipTo = "-1"                       # 9 Product_ShippedTo
-    left = "-1"                         # 10 Product_QuantityLeft
-    escrow = "-1"                       # 11 Vendor_Warranty
-    terms = "-1"                        # 12 Vendor_TermsAndConditions
-    vendor = "-1"                       # 13 Vendor_Name
-    sold = "-1"                         # 14 Product_QuantitySold
-    addDate = "-1"                      # 15 Product_AddedDate
-    available = "-1"                    # 16 NOT USED ...
-    endDate = "-1"                      # 17 NOT USED ...
-    BTC = "-1"                          # 18 Product_BTC_SellingPrice
-    USD = "-1"                          # 19 Product_USD_SellingPrice
-    rating = "-1"                       # 20 Vendor_Rating
-    success = "-1"                      # 21 Vendor_Successful_Transactions
-    EURO = "-1"                         # 22 Product_EURO_SellingPrice
-
-    # Finding Product Name
-    name = soup.find('h1').text
-    name = name.replace('\n', ' ')
-    name = name.replace(",", "")
-    name = name.strip()
-
-    # Finding Vendor
-    vendor = soup.find('h3').find('a').text.strip()
-
-    # Finding Vendor Rating
-    rating = soup.find('span', {'class': "tag is-dark"}).text.strip()
-
-    # Finding Successful Transactions
-    success = soup.find('h3').text
-    success = success.replace("Vendor: ", "")
-    success = success.replace(vendor, "")
-    success = success.replace("(", "")
-    success = success.replace(")", "")
-    success = success.strip()
-
-    bae = soup.find('div', {'class': "box"}).find_all('ul')
-
-    # Finding Prices
-    USD = bae[1].find('strong').text.strip()
-
-    li = bae[2].find_all('li')
-
-    # Finding Escrow
-    escrow = li[0].find('span', {'class': "tag is-dark"}).text.strip()
-
-    # Finding the Product Category
-    category = li[1].find('span', {'class': "tag is-dark"}).text.strip()
-
-    # Finding the Product Quantity Available
-    left = li[3].find('span', {'class': "tag is-dark"}).text.strip()
-
-    # Finding Number Sold
-    sold = li[4].find('span', {'class': "tag is-dark"}).text.strip()
-
-    li = bae[3].find_all('li')
-
-    # Finding Shipment Information (Origin)
-    if "Ships from:" in li[-2].text:
-        shipFrom = li[-2].text
-        shipFrom = shipFrom.replace("Ships from: ", "")
-        # shipFrom = shipFrom.replace(",", "")
-        shipFrom = shipFrom.strip()
-
-    # Finding Shipment Information (Destination)
-    shipTo = li[-1].find('div', {'title': "List of countries is scrollable"}).text
-    shipTo = shipTo.replace("Ships to: ", "")
-    shipTo = shipTo.strip()
-    if "certain countries" in shipTo:
-        countries = ""
-        tags = li[-1].find_all('span', {'class': "tag"})
-        for tag in tags:
-            country = tag.text.strip()
-            countries += country + ", "
-        shipTo = countries.strip(", ")
-
-    # Finding the Product description
-    describe = soup.find('div', {'class': "pre-line"}).text
-    describe = describe.replace("\n", " ")
-    describe = describe.strip()
-
-    '''# Finding the Number of Product Reviews
-    tag = soup.findAll(text=re.compile('Reviews'))
-    for index in tag:
-        reviews = index
-        par = reviews.find('(')
-        if par >=0:
-            reviews = reviews.replace("Reviews (","")
-            reviews = reviews.replace(")","")
-            reviews = reviews.split(",")
-            review = str(abs(int(reviews[0])) + abs(int(reviews[1])))
-        else  :
-            review = "-1"'''
-
-    # Searching for CVE and MS categories
-    cve = soup.findAll(text=re.compile('CVE-\d{4}-\d{4}'))
-    if cve:
-        CVE = " "
-        for idx in cve:
-            CVE += (idx)
-            CVE += "  "
-            CVE = CVE.replace(',', ' ')
-            CVE = CVE.replace('\n', '')
-    ms = soup.findAll(text=re.compile('MS\d{2}-\d{3}'))
-    if ms:
-        MS = " "
-        for im in ms:
-            MS += (im)
-            MS += " "
-            MS = MS.replace(',', ' ')
-            MS = MS.replace('\n', '')
+    vendor = "-1"  # 0 *Vendor_Name
+    success = "-1"  # 1 Vendor_Successful_Transactions
+    rating_vendor = "-1"  # 2 Vendor_Rating
+    name = "-1"  # 3 *Product_Name
+    describe = "-1"  # 4 Product_Description
+    CVE = "-1"  # 5 Product_CVE_Classification (Common Vulnerabilities and Exposures)
+    MS = "-1"  # 6 Product_MS_Classification (Microsoft Security)
+    category = "-1"  # 7 Product_Category
+    views = "-1"  # 8 Product_Number_Of_Views
+    reviews = "-1"  # 9 Product_Number_Of_Reviews
+    rating_item = "-1"  # 10 Product_Rating
+    addDate = "-1"  # 11 Product_AddedDate
+    BTC = "-1"  # 12 Product_BTC_SellingPrice
+    USD = "-1"  # 13 Product_USD_SellingPrice
+    EURO = "-1"  # 14 Product_EURO_SellingPrice
+    sold = "-1"  # 15 Product_QuantitySold
+    left = "-1"  # 16 Product_QuantityLeft
+    shipFrom = "-1"  # 17 Product_ShippedFrom
+    shipTo = "-1"  # 18 Product_ShippedTo
+
+    vendor = "-1"  # 0 *Vendor_Name
+    success = "-1"  # 1 Vendor_Successful_Transactions
+    rating_vendor = "-1"  # 2 Vendor_Rating
+
+
+    # product name
+    try:
+        name = soup.find('head').find('title').text
+        name = cleanString(name.strip())
+    except:
+        print("name")
+
+    #product description
+    try:
+        temp = soup.find('pre', {'class', 'description'}).text
+        temp = temp.replace('\n', ' ')
+        describe = cleanString(temp.strip())
+    except:
+        print("description")
+
+
+
+    CVE = "-1"  # 5 Product_CVE_Classification (Common Vulnerabilities and Exposures)
+    MS = "-1"  # 6 Product_MS_Classification (Microsoft Security)
+    category = "-1"  # 7 Product_Category
+    views = "-1"  # 8 Product_Number_Of_Views
+    reviews = "-1"  # 9 Product_Number_Of_Reviews
+    rating_item = "-1"  # 10 Product_Rating
+    addDate = "-1"  # 11 Product_AddedDate
+    BTC = "-1"  # 12 Product_BTC_SellingPrice
+    USD = "-1"  # 13 Product_USD_SellingPrice
+    EURO = "-1"  # 14 Product_EURO_SellingPrice
+    sold = "-1"  # 15 Product_QuantitySold
+    left = "-1"  # 16 Product_QuantityLeft
+    shipFrom = "-1"  # 17 Product_ShippedFrom
+    shipTo = "-1"  # 18 Product_ShippedTo
 
     # Populating the final variable (this should be a list with all fields scraped)
-    row = (name, describe, lastSeen, rules, CVE, MS, review, category, shipFrom, shipTo, left, escrow, terms, vendor,
-       sold, addDate, available, endDate, BTC, USD, rating, success, EURO)
+    row = (vendor, rating_vendor, success, name, describe, CVE, MS, category, views, reviews, rating_item, addDate,
+           BTC, USD, EURO, sold, left, shipFrom, shipTo)
 
     # Sending the results
     return row
@@ -147,11 +84,11 @@ def darkfox_description_parser(soup):
 #stores info it needs in different lists, these lists are returned after being organized
 #@param: soup object looking at html page of listing page
 #return: 'row' that contains a variety of lists that each hold info on the listing page
-def darkfox_listing_parser(soup):
+def darkmatter_listing_parser(soup):
 
     # Fields to be parsed
     nm = 0                                    # Total_Products (Should be Integer)
-    mktName = "DarkFox"                       # 0 Marketplace_Name
+    mktName = "DarkMatter"                       # 0 Marketplace_Name
     name = []                                 # 1 Product_Name
     CVE = []                                  # 2 Product_CVE_Classification (Common Vulnerabilities and Exposures)
     MS = []                                   # 3 Product_MS_Classification (Microsoft Security)
@@ -281,7 +218,7 @@ def darkmatter_links_parser(soup):
     # Returning all links that should be visited by the Crawler
 
     href = []
-    listing = soup.findAll('td', {"class": "lefted", 'colspan': '2'})
+    listing = soup.find('div', {"class": "content"}).findAll('td', {"class": "lefted", 'colspan': '3'})
 
     for a in listing:
         bae = a.find('a', href=True)

MarketPlaces/Initialization/geckodriver.log

@@ -15746,3 +15746,463 @@ unwatchForTargets()@TargetList.jsm:37
 destructor()@TargetList.jsm:109
 stop()@CDP.jsm:104
 close()@RemoteAgent.jsm:138
+1689709756057	geckodriver	INFO	Listening on 127.0.0.1:51663
+1689709760560	mozrunner::runner	INFO	Running command: "C:\\Users\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "51664" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofilecnoWYx"
+console.log: "TorSettings: loadFromPrefs()"
+console.log: "TorConnect: init()"
+console.log: "TorConnect: Entering Initial state"
+console.log: "TorConnect: Observed profile-after-change"
+console.log: "TorConnect: Observing topic 'TorProcessExited'"
+console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'"
+console.log: "TorConnect: Observing topic 'torsettings:ready'"
+console.log: "TorSettings: Observed profile-after-change"
+1689709761952	Marionette	INFO	Marionette enabled
+console.log: "TorConnect: Will load after bootstrap => [about:blank]"
+console.error: "Could not load engine [email protected]: Error: Extension is invalid"
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
+DevTools listening on ws://localhost:51664/devtools/browser/f768056e-c403-4ca5-a40a-2be0342d7786
+1689709765202	Marionette	INFO	Listening on port 51669
+1689709765605	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+1689709770084	Marionette	INFO	Stopped listening on port 51669
+JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
+!!! error running onStopped callback: TypeError: callback is not a function
+JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first.
+[Parent 7656, IPC I/O Parent] WARNING: file /var/tmp/build/firefox-b6010b1466c9/ipc/chromium/src/base/process_util_win.cc:167
+JavaScript error: resource://gre/modules/PageThumbs.jsm, line 709: AbortError: IOUtils.profileBeforeChange getter: IOUtils: profileBeforeChange phase has already finished
+
+###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost
+
+1689709771175	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
+unwatchForTabs()@TargetList.jsm:70
+unwatchForTargets()@TargetList.jsm:37
+destructor()@TargetList.jsm:109
+stop()@CDP.jsm:104
+close()@RemoteAgent.jsm:138
+1689710063743	geckodriver	INFO	Listening on 127.0.0.1:51697
+1689710068542	mozrunner::runner	INFO	Running command: "C:\\Users\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "51698" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofileOFhAta"
+console.log: "TorSettings: loadFromPrefs()"
+console.log: "TorConnect: init()"
+console.log: "TorConnect: Entering Initial state"
+console.log: "TorConnect: Observed profile-after-change"
+console.log: "TorConnect: Observing topic 'TorProcessExited'"
+console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'"
+console.log: "TorConnect: Observing topic 'torsettings:ready'"
+console.log: "TorSettings: Observed profile-after-change"
+1689710069671	Marionette	INFO	Marionette enabled
+console.log: "TorConnect: Will load after bootstrap => [about:blank]"
+console.error: "Could not load engine [email protected]: Error: Extension is invalid"
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
+DevTools listening on ws://localhost:51698/devtools/browser/ea7fcb35-4ed9-4198-a004-9afa8f47449e
+1689710072576	Marionette	INFO	Listening on port 51703
+[Parent 8820, IPC I/O Parent] WARNING: file /var/tmp/build/firefox-b6010b1466c9/ipc/chromium/src/base/process_util_win.cc:167
+1689710072872	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+1689710076807	Marionette	INFO	Stopped listening on port 51703
+JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
+JavaScript error: resource://gre/modules/AsyncShutdown.jsm, line 575: AbortError: Actor 'Conduits' destroyed before query 'RunListener' was resolved
+!!! error running onStopped callback: TypeError: callback is not a function
+JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first.
+JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofileOFhAta\thumbnails) because it does not exist
+1689710078072	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
+unwatchForTabs()@TargetList.jsm:70
+unwatchForTargets()@TargetList.jsm:37
+destructor()@TargetList.jsm:109
+stop()@CDP.jsm:104
+close()@RemoteAgent.jsm:138
+1689710459622	geckodriver	INFO	Listening on 127.0.0.1:51728
+1689710464469	mozrunner::runner	INFO	Running command: "C:\\Users\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "51729" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofile4MFS1M"
+console.log: "TorSettings: loadFromPrefs()"
+console.log: "TorConnect: init()"
+console.log: "TorConnect: Entering Initial state"
+console.log: "TorConnect: Observed profile-after-change"
+console.log: "TorConnect: Observing topic 'TorProcessExited'"
+console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'"
+console.log: "TorConnect: Observing topic 'torsettings:ready'"
+console.log: "TorSettings: Observed profile-after-change"
+1689710465911	Marionette	INFO	Marionette enabled
+console.log: "TorConnect: Will load after bootstrap => [about:blank]"
+console.error: "Could not load engine [email protected]: Error: Extension is invalid"
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
+DevTools listening on ws://localhost:51729/devtools/browser/f2a01690-36e9-4b89-a771-eff57abbfbab
+1689710469141	Marionette	INFO	Listening on port 51734
+1689710469560	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+1689710474490	Marionette	INFO	Stopped listening on port 51734
+JavaScript error: resource://gre/modules/AsyncShutdown.jsm, line 575: NotFoundError: No such JSWindowActor 'MarionetteEvents'
+JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
+!!! error running onStopped callback: TypeError: callback is not a function
+JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first.
+JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofile4MFS1M\thumbnails) because it does not exist
+[Parent 1508, IPC I/O Parent] WARNING: file /var/tmp/build/firefox-b6010b1466c9/ipc/chromium/src/base/process_util_win.cc:167
+
+###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost
+
+1689710475545	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
+unwatchForTabs()@TargetList.jsm:70
+unwatchForTargets()@TargetList.jsm:37
+destructor()@TargetList.jsm:109
+stop()@CDP.jsm:104
+close()@RemoteAgent.jsm:138
+1689710757807	geckodriver	INFO	Listening on 127.0.0.1:51815
+1689710764025	mozrunner::runner	INFO	Running command: "C:\\Users\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "51816" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofileIiqjFc"
+console.log: "TorSettings: loadFromPrefs()"
+console.log: "TorConnect: init()"
+console.log: "TorConnect: Entering Initial state"
+console.log: "TorConnect: Observed profile-after-change"
+console.log: "TorConnect: Observing topic 'TorProcessExited'"
+console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'"
+console.log: "TorConnect: Observing topic 'torsettings:ready'"
+console.log: "TorSettings: Observed profile-after-change"
+1689710766088	Marionette	INFO	Marionette enabled
+console.log: "TorConnect: Will load after bootstrap => [about:blank]"
+console.error: "Could not load engine [email protected]: Error: Extension is invalid"
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
+DevTools listening on ws://localhost:51816/devtools/browser/58b205d5-0d91-4445-9918-a1d9ecad5142
+1689710770478	Marionette	INFO	Listening on port 51823
+1689710770972	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+1689710777368	Marionette	INFO	Stopped listening on port 51823
+JavaScript error: resource://gre/modules/AsyncShutdown.jsm, line 575: NotFoundError: No such JSWindowActor 'MarionetteEvents'
+JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
+!!! error running onStopped callback: TypeError: callback is not a function
+JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first.
+JavaScript error: resource://gre/modules/PageThumbs.jsm, line 709: AbortError: IOUtils.profileBeforeChange getter: IOUtils: profileBeforeChange phase has already finished
+1689710778787	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
+unwatchForTabs()@TargetList.jsm:70
+unwatchForTargets()@TargetList.jsm:37
+destructor()@TargetList.jsm:109
+stop()@CDP.jsm:104
+close()@RemoteAgent.jsm:138
+1689710937611	geckodriver	INFO	Listening on 127.0.0.1:51918
+1689710943129	mozrunner::runner	INFO	Running command: "C:\\Users\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "51919" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofile2o2xXA"
+console.log: "TorSettings: loadFromPrefs()"
+console.log: "TorConnect: init()"
+console.log: "TorConnect: Entering Initial state"
+console.log: "TorConnect: Observed profile-after-change"
+console.log: "TorConnect: Observing topic 'TorProcessExited'"
+console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'"
+console.log: "TorConnect: Observing topic 'torsettings:ready'"
+console.log: "TorSettings: Observed profile-after-change"
+1689710944863	Marionette	INFO	Marionette enabled
+console.log: "TorConnect: Will load after bootstrap => [about:blank]"
+console.error: "Could not load engine [email protected]: Error: Extension is invalid"
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
+DevTools listening on ws://localhost:51919/devtools/browser/0ea37d47-ce80-46f8-98ac-2e8422e52e4a
+1689710948643	Marionette	INFO	Listening on port 51926
+1689710948794	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+1689710954589	Marionette	INFO	Stopped listening on port 51926
+JavaScript error: resource://gre/modules/AsyncShutdown.jsm, line 575: NotFoundError: No such JSWindowActor 'MarionetteEvents'
+JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
+JavaScript error: resource://gre/modules/AsyncShutdown.jsm, line 575: AbortError: Actor 'Conduits' destroyed before query 'RunListener' was resolved
+JavaScript error: , line 0: AbortError: Actor 'TorConnect' destroyed before query 'torconnect:get-init-args' was resolved
+[Parent 5144, IPC I/O Parent] WARNING: file /var/tmp/build/firefox-b6010b1466c9/ipc/chromium/src/base/process_util_win.cc:167
+!!! error running onStopped callback: TypeError: callback is not a function
+JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first.
+JavaScript error: resource://gre/modules/PageThumbs.jsm, line 709: AbortError: IOUtils.profileBeforeChange getter: IOUtils: profileBeforeChange phase has already finished
+[Parent 5144, IPC I/O Parent] WARNING: file /var/tmp/build/firefox-b6010b1466c9/ipc/chromium/src/base/process_util_win.cc:167
+1689710955786	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
+unwatchForTabs()@TargetList.jsm:70
+unwatchForTargets()@TargetList.jsm:37
+destructor()@TargetList.jsm:109
+stop()@CDP.jsm:104
+close()@RemoteAgent.jsm:138
+1689711278426	geckodriver	INFO	Listening on 127.0.0.1:52091
+1689711285611	mozrunner::runner	INFO	Running command: "C:\\Users\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "52092" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofile8PhBso"
+console.log: "TorSettings: loadFromPrefs()"
+console.log: "TorConnect: init()"
+console.log: "TorConnect: Entering Initial state"
+console.log: "TorConnect: Observed profile-after-change"
+console.log: "TorConnect: Observing topic 'TorProcessExited'"
+console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'"
+console.log: "TorConnect: Observing topic 'torsettings:ready'"
+console.log: "TorSettings: Observed profile-after-change"
+1689711287763	Marionette	INFO	Marionette enabled
+console.log: "TorConnect: Will load after bootstrap => [about:blank]"
+console.error: "Could not load engine [email protected]: Error: Extension is invalid"
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
+DevTools listening on ws://localhost:52092/devtools/browser/c4e8eb6c-b8f8-48db-9575-d8b57e35ce4d
+1689711292809	Marionette	INFO	Listening on port 52099
+1689711293004	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+1689711300299	Marionette	INFO	Stopped listening on port 52099
+JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
+JavaScript error: resource://gre/modules/AsyncShutdown.jsm, line 575: AbortError: Actor 'Conduits' destroyed before query 'RunListener' was resolved
+!!! error running onStopped callback: TypeError: callback is not a function
+JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first.
+JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofile8PhBso\thumbnails) because it does not exist
+1689711301829	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
+unwatchForTabs()@TargetList.jsm:70
+unwatchForTargets()@TargetList.jsm:37
+destructor()@TargetList.jsm:109
+stop()@CDP.jsm:104
+close()@RemoteAgent.jsm:138
+1689711885737	geckodriver	INFO	Listening on 127.0.0.1:52388
+1689711893670	mozrunner::runner	INFO	Running command: "C:\\Users\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "52389" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofileu9OZn2"
+console.log: "TorSettings: loadFromPrefs()"
+console.log: "TorConnect: init()"
+console.log: "TorConnect: Entering Initial state"
+console.log: "TorConnect: Observed profile-after-change"
+console.log: "TorConnect: Observing topic 'TorProcessExited'"
+console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'"
+console.log: "TorConnect: Observing topic 'torsettings:ready'"
+console.log: "TorSettings: Observed profile-after-change"
+1689711895921	Marionette	INFO	Marionette enabled
+console.log: "TorConnect: Will load after bootstrap => [about:blank]"
+console.error: "Could not load engine [email protected]: Error: Extension is invalid"
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
+DevTools listening on ws://localhost:52389/devtools/browser/35447fb8-4ebf-4caf-8c89-13038fc79d2b
+1689711901057	Marionette	INFO	Listening on port 52396
+1689711901777	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+1689711909117	Marionette	INFO	Stopped listening on port 52396
+JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
+!!! error running onStopped callback: TypeError: callback is not a function
+JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first.
+JavaScript error: resource://gre/modules/PageThumbs.jsm, line 709: AbortError: IOUtils.profileBeforeChange getter: IOUtils: profileBeforeChange phase has already finished
+[Parent 9012, IPC I/O Parent] WARNING: file /var/tmp/build/firefox-b6010b1466c9/ipc/chromium/src/base/process_util_win.cc:167
+1689711910554	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
+unwatchForTabs()@TargetList.jsm:70
+unwatchForTargets()@TargetList.jsm:37
+destructor()@TargetList.jsm:109
+stop()@CDP.jsm:104
+close()@RemoteAgent.jsm:138
+1689712165507	geckodriver	INFO	Listening on 127.0.0.1:52569
+1689712171396	mozrunner::runner	INFO	Running command: "C:\\Users\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "52570" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofileBYUxGv"
+console.log: "TorSettings: loadFromPrefs()"
+console.log: "TorConnect: init()"
+console.log: "TorConnect: Entering Initial state"
+console.log: "TorConnect: Observed profile-after-change"
+console.log: "TorConnect: Observing topic 'TorProcessExited'"
+console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'"
+console.log: "TorConnect: Observing topic 'torsettings:ready'"
+console.log: "TorSettings: Observed profile-after-change"
+1689712173264	Marionette	INFO	Marionette enabled
+console.log: "TorConnect: Will load after bootstrap => [about:blank]"
+console.error: "Could not load engine [email protected]: Error: Extension is invalid"
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
+DevTools listening on ws://localhost:52570/devtools/browser/46c17946-d8da-4f79-89da-b48961358c5e
+1689712177652	Marionette	INFO	Listening on port 52577
+1689712178244	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+1689712395837	Marionette	INFO	Stopped listening on port 52577
+JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
+JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData]
+!!! error running onStopped callback: TypeError: callback is not a function
+JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first.
+JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofileBYUxGv\thumbnails) because it does not exist
+[Parent 6172, IPC I/O Parent] WARNING: file /var/tmp/build/firefox-b6010b1466c9/ipc/chromium/src/base/process_util_win.cc:167
+1689712396253	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
+unwatchForTabs()@TargetList.jsm:70
+unwatchForTargets()@TargetList.jsm:37
+destructor()@TargetList.jsm:109
+stop()@CDP.jsm:104
+close()@RemoteAgent.jsm:138
+1689712458291	geckodriver	INFO	Listening on 127.0.0.1:52745
+1689712464089	mozrunner::runner	INFO	Running command: "C:\\Users\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "52746" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofile7pa8h1"
+console.log: "TorSettings: loadFromPrefs()"
+console.log: "TorConnect: init()"
+console.log: "TorConnect: Entering Initial state"
+console.log: "TorConnect: Observed profile-after-change"
+console.log: "TorConnect: Observing topic 'TorProcessExited'"
+console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'"
+console.log: "TorConnect: Observing topic 'torsettings:ready'"
+console.log: "TorSettings: Observed profile-after-change"
+1689712465452	Marionette	INFO	Marionette enabled
+console.log: "TorConnect: Will load after bootstrap => [about:blank]"
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+console.error: "Could not load engine [email protected]: Error: Extension is invalid"
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
+DevTools listening on ws://localhost:52746/devtools/browser/f1aa2cdb-25e1-4240-992b-5dc17ec4b35a
+1689712468928	Marionette	INFO	Listening on port 52754
+1689712469432	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+JavaScript error: resource://gre/actors/AutoCompleteChild.jsm, line 125: Error: Invalid autocomplete selectedIndex
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+1689712803150	Marionette	INFO	Stopped listening on port 52754
+JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
+!!! error running onStopped callback: TypeError: callback is not a function
+JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first.
+JavaScript error: resource://gre/modules/PageThumbs.jsm, line 709: AbortError: IOUtils.profileBeforeChange getter: IOUtils: profileBeforeChange phase has already finished
+
+###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost
+
+1689712803626	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
+unwatchForTabs()@TargetList.jsm:70
+unwatchForTargets()@TargetList.jsm:37
+destructor()@TargetList.jsm:109
+stop()@CDP.jsm:104
+close()@RemoteAgent.jsm:138
+1689725884014	geckodriver	INFO	Listening on 127.0.0.1:57758
+1689725890715	mozrunner::runner	INFO	Running command: "C:\\Users\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "57759" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofileKZtDvb"
+console.log: "TorSettings: loadFromPrefs()"
+console.log: "TorConnect: init()"
+console.log: "TorConnect: Entering Initial state"
+console.log: "TorConnect: Observed profile-after-change"
+console.log: "TorConnect: Observing topic 'TorProcessExited'"
+console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'"
+console.log: "TorConnect: Observing topic 'torsettings:ready'"
+console.log: "TorSettings: Observed profile-after-change"
+1689725894724	Marionette	INFO	Marionette enabled
+console.log: "TorConnect: Will load after bootstrap => [about:blank]"
+JavaScript error: resource:///modules/UrlbarInput.jsm, line 2952: TypeError: event.target.closest is not a function
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+console.error: "Could not load engine [email protected]: Error: Extension is invalid"
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
+DevTools listening on ws://localhost:57759/devtools/browser/5130c45f-e57f-4719-970e-f1ca80613eda
+1689725901218	Marionette	INFO	Listening on port 57767
+1689725902070	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+
+###!!! [Child][MessageChannel] Error: (msgtype=0x390097,name=PContent::Msg_InitBackground) Channel closing: too late to send/recv, messages will be lost
+
+1689725909826	Marionette	INFO	Stopped listening on port 57767
+JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
+JavaScript error: resource://gre/modules/AsyncShutdown.jsm, line 575: AbortError: Actor 'Conduits' destroyed before query 'RunListener' was resolved
+!!! error running onStopped callback: TypeError: callback is not a function
+JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first.
+JavaScript error: resource://gre/modules/PageThumbs.jsm, line 709: AbortError: IOUtils.profileBeforeChange getter: IOUtils: profileBeforeChange phase has already finished
+[Parent 4420, IPC I/O Parent] WARNING: file /var/tmp/build/firefox-b6010b1466c9/ipc/chromium/src/base/process_util_win.cc:167
+1689725911309	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
+unwatchForTabs()@TargetList.jsm:70
+unwatchForTargets()@TargetList.jsm:37
+destructor()@TargetList.jsm:109
+stop()@CDP.jsm:104
+close()@RemoteAgent.jsm:138
+1689726298926	geckodriver	INFO	Listening on 127.0.0.1:57942
+1689726305572	mozrunner::runner	INFO	Running command: "C:\\Users\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "57943" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofile4REPVd"
+console.log: "TorSettings: loadFromPrefs()"
+console.log: "TorConnect: init()"
+console.log: "TorConnect: Entering Initial state"
+console.log: "TorConnect: Observed profile-after-change"
+console.log: "TorConnect: Observing topic 'TorProcessExited'"
+console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'"
+console.log: "TorConnect: Observing topic 'torsettings:ready'"
+console.log: "TorSettings: Observed profile-after-change"
+1689726307885	Marionette	INFO	Marionette enabled
+console.log: "TorConnect: Will load after bootstrap => [about:blank]"
+console.error: "Could not load engine [email protected]: Error: Extension is invalid"
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
+DevTools listening on ws://localhost:57943/devtools/browser/77548608-482e-4f84-83b5-ea9ed5de1e6c
+1689726313342	Marionette	INFO	Listening on port 57950
+1689726313710	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+1689726526374	Marionette	INFO	Stopped listening on port 57950
+JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
+!!! error running onStopped callback: TypeError: callback is not a function
+JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first.
+JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofile4REPVd\thumbnails) because it does not exist
+[Parent 6328, IPC I/O Parent] WARNING: file /var/tmp/build/firefox-b6010b1466c9/ipc/chromium/src/base/process_util_win.cc:167
+[Parent 6328, IPC I/O Parent] WARNING: pipe error: 232: file /var/tmp/build/firefox-b6010b1466c9/ipc/chromium/src/chrome/common/ipc_channel_win.cc:544
+1689726527913	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
+unwatchForTabs()@TargetList.jsm:70
+unwatchForTargets()@TargetList.jsm:37
+destructor()@TargetList.jsm:109
+stop()@CDP.jsm:104
+close()@RemoteAgent.jsm:138
+1689727835682	geckodriver	INFO	Listening on 127.0.0.1:58463
+1689727842340	mozrunner::runner	INFO	Running command: "C:\\Users\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "58464" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofileSzsJi7"
+console.log: "TorSettings: loadFromPrefs()"
+console.log: "TorConnect: init()"
+console.log: "TorConnect: Entering Initial state"
+console.log: "TorConnect: Observed profile-after-change"
+console.log: "TorConnect: Observing topic 'TorProcessExited'"
+console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'"
+console.log: "TorConnect: Observing topic 'torsettings:ready'"
+console.log: "TorSettings: Observed profile-after-change"
+1689727844481	Marionette	INFO	Marionette enabled
+console.log: "TorConnect: Will load after bootstrap => [about:blank]"
+console.error: "Could not load engine [email protected]: Error: Extension is invalid"
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined
+DevTools listening on ws://localhost:58464/devtools/browser/d34aa7cb-51f8-41fe-9e31-7a0ac5675ef9
+1689727850376	Marionette	INFO	Listening on port 58471
+1689727851163	RemoteAgent	WARN	TLS certificate errors will be ignored for this session
+JavaScript error: resource://gre/actors/AutoCompleteChild.jsm, line 125: Error: Invalid autocomplete selectedIndex
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory.
+SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/b0ecb48a-4f1b-4898-8d6c-86f25b19102b'"  nsresult: "0x805303f4 (<unknown>)"  location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670"  data: yes]
+Stack: mainThreadFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15
+mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10
+_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22
+urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17
+_resurrectSource@resource://devtools/server/actors/thread.js:2142:35
+addAllSources@resource://devtools/server/actors/thread.js:1509:14
+watch@resource://devtools/server/actors/resources/sources.js:52:17
+watchResources@resource://devtools/server/actors/resources/index.js:239:19
+_watchTargetResources@resource://devtools/server/actors/targets/target-actor-mixin.js:156:24
+addWatcherDataEntry@resource://devtools/server/actors/targets/target-actor-mixin.js:47:20
+_addWatcherDataEntry@resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm:483:24
+receiveMessage@resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm:425:21
+Line: 670, column: 0
+console.error: ({})
+SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/37f88ac5-4cdf-4bdc-b792-480b4f61196c'"  nsresult: "0x805303f4 (<unknown>)"  location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670"  data: yes]
+Stack: mainThreadFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15
+mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10
+_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22
+urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17
+_resurrectSource@resource://devtools/server/actors/thread.js:2142:35
+addAllSources@resource://devtools/server/actors/thread.js:1509:14
+watch@resource://devtools/server/actors/resources/sources.js:52:17
+watchResources@resource://devtools/server/actors/resources/index.js:239:19
+_watchTargetResources@resource://devtools/server/actors/targets/target-actor-mixin.js:156:24
+addWatcherDataEntry@resource://devtools/server/actors/targets/target-actor-mixin.js:47:20
+_addWatcherDataEntry@resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm:483:24
+receiveMessage@resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm:425:21
+Line: 670, column: 0
+console.error: ({})
+JavaScript error: resource://devtools/shared/DevToolsUtils.js, line 670: Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/b0ecb48a-4f1b-4898-8d6c-86f25b19102b'
+JavaScript error: resource://devtools/shared/DevToolsUtils.js, line 670: Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/37f88ac5-4cdf-4bdc-b792-480b4f61196c'
+1689728550247	Marionette	INFO	Stopped listening on port 58471
+JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver]
+JavaScript error: resource:///actors/RFPHelperParent.jsm, line 28: TypeError: browser is null
+JavaScript error: resource:///actors/RFPHelperParent.jsm, line 28: TypeError: browser is null
+JavaScript error: resource:///actors/RFPHelperParent.jsm, line 28: TypeError: browser is null
+!!! error running onStopped callback: TypeError: callback is not a function
+JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first.
+JavaScript error: resource://gre/modules/PageThumbs.jsm, line 709: AbortError: IOUtils.profileBeforeChange getter: IOUtils: profileBeforeChange phase has already finished
+[Parent 5584, IPC I/O Parent] WARNING: file /var/tmp/build/firefox-b6010b1466c9/ipc/chromium/src/base/process_util_win.cc:167
+1689728551276	RemoteAgent	ERROR	unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]"  nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)"  location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64"  data: no] Stack trace: stop()@TargetObserver.jsm:64
+unwatchForTabs()@TargetList.jsm:70
+unwatchForTargets()@TargetList.jsm:37
+destructor()@TargetList.jsm:109
+stop()@CDP.jsm:104
+close()@RemoteAgent.jsm:138

MarketPlaces/Initialization/prepare_parser.py

@@ -22,7 +22,7 @@ def mergePages(rmm, rec):
     # key = u"Pr:" + rec[1].upper() + u" Vendor:" + rec[18].upper()
     # key = rec[23]
 
-    print("----------------- Matched: " + rec[1] + "--------------------")
+    print("----------------- Matched: " + rec[4] + "--------------------")
 
     if rec[1] == "-1":          # name_vendor
         rec[1] = rmm[0]
@@ -263,7 +263,6 @@ def new_parse(marketPlace, url, createLog):
                             persist_data(url, tuple(rec), cur)
                             con.commit()
                         except Exception as e:
-                            raise e
 
                             trace = traceback.format_exc()
 

MarketPlaces/M00nkeyMarket/crawler_selenium.py

@@ -35,7 +35,7 @@ MARKET_NAME = 'M00nkeyMarket'
 def startCrawling():
     opentor()
     driver = getAccess()
-    
+
     if driver != 'down':
         try:
             login(driver)
@@ -203,7 +203,7 @@ def getInterestedLinks():
     # software
     links.append('http://moonkey4f2mkcp6hpackeea356puiry27h3dz3hzbt3adbmsk4gs7wyd.onion/search/subcategories?subcategory=30')
     # # guides
-    links.append('http://moonkey4f2mkcp6hpackeea356puiry27h3dz3hzbt3adbmsk4gs7wyd.onion/search/subcategories?subcategory=17')
+    # links.append('http://moonkey4f2mkcp6hpackeea356puiry27h3dz3hzbt3adbmsk4gs7wyd.onion/search/subcategories?subcategory=17')
 
     return links
 
@@ -243,11 +243,11 @@ def crawlForum(driver):
                     driver.back()
 
                     # comment out
-                    break
+                    # break
 
                 # comment out
-                if count == 1:
-                    break
+                # if count == 1:
+                #     break
 
                 try:
                     link = driver.find_element(by=By.LINK_TEXT, value='Next ›').get_attribute('href')

MarketPlaces/M00nkeyMarket/parser.py

@@ -35,26 +35,38 @@ def m00nkey_description_parser(soup):
     shipTo = "-1"  # 18 Product_ShippedTo
 
     #vendor name
-    temp = soup.find('div', {'class': 'box rounded mb-0'}).find('a').text
-    vendor = (cleanString(temp.strip()))
+    try:
+        temp = soup.find('div', {'class': 'box rounded mb-0'}).find('a').text
+        vendor = (cleanString(temp.strip()))
+    except:
+        vendor = "-1"
 
     #successful transaction
-    temp = soup.findAll('div', {'class','text-center text-truncate column-flex ml-1 mr-1'}) #card sidebar-menu mb-4 card sidebar-menu mb-4
-    temp2 = temp[1].findAll('span', {'class', 'float-right font-weight-bold'})
-    temp = temp2[1].text
-    success = (cleanString(temp.strip()))
+    try:
+        temp = soup.findAll('div', {'class','text-center text-truncate column-flex ml-1 mr-1'}) #card sidebar-menu mb-4 card sidebar-menu mb-4
+        temp2 = temp[1].findAll('span', {'class', 'float-right font-weight-bold'})
+        temp = temp2[1].text
+        success = (temp.strip())
+    except:
+        print("success")
 
 
     #vendor rating 5
-    temp = soup.findAll('div', {'class', 'text-center text-truncate column-flex ml-1 mr-1'})  # card sidebar-menu mb-4 card sidebar-menu mb-4
-    temp2 = temp[1].findAll('span', {'class', 'float-right font-weight-bold'})
-    temp = temp2[5].text
-    rating_vendor = (cleanString(temp.strip()))
+    try:
+        temp = soup.findAll('div', {'class', 'text-center text-truncate column-flex ml-1 mr-1'})  # card sidebar-menu mb-4 card sidebar-menu mb-4
+        temp2 = temp[1].findAll('span', {'class', 'float-right font-weight-bold'})
+        temp = temp2[5].text
+        rating_vendor = (cleanString(temp.strip()))
+    except:
+        print("product")
 
     # product name
-    temp = soup.find('h3', {'class', 'h3 rounded card-title'}).find('span').text
-    name = (cleanString(temp.strip()))
-
+    try:
+        temp = soup.find('h3', {'class', 'h3 rounded card-title'}).find('span').text
+        name = (cleanString(temp.strip()))
+    except:
+        temp = soup.find('h3', {'class', 'h3 rounded card-title'}).find('span').find("div").text
+        name = (cleanString(temp.strip()))
 
     # product description
     describe = soup.find('div', {'class': "box rounded flex-fill"}).find('pre').text
@@ -86,35 +98,58 @@ def m00nkey_description_parser(soup):
         views = cleanString((temp.strip()))
     except:
         print('Product number of view')
-        # views = "-1"
 
     reviews = "-1"  # 9 Product_Number_Of_Reviews
     rating_item = "-1"  # 10 Product_Rating
     addDate = "-1"  # 11 Product_AddedDate
 
     #BTC selling price box box-rounded mt-2
-    temp = soup.find('div', {'class', 'box box-rounded mt-2'})
-    temp2 = temp.findAll('i', {'class', 'float-right color-prices'})
-    temp = temp2[1].text
-    BTC = cleanString((temp.strip()))
+    try:
+        temp = soup.find('div', {'class', 'box box-rounded mt-2'})
+        temp2 = temp.findAll('i', {'class', 'float-right color-prices'})
+        temp = temp2[1].text
+        BTC = cleanString((temp.strip()))
+    except:
+        try:
+            temp = soup.find('div', {'class', 'box box-rounded'})
+            temp2 = temp.findAll('span', {'class', 'float-right color-prices'})
+            temp = temp2[1].text
+            BTC = cleanString((temp.strip()))
+        except:
+            print("BTC")
+
 
     # USD selling price
-    temp = soup.find('div', {'class', 'box box-rounded mt-2'})
-    temp2 = temp.findAll('center')
-    temp = temp2[1].find('i').text
-    if "$" in temp:
-        temp = temp.replace("$", "")
-    USD = cleanString((temp.strip()))
+    try:
+        temp = soup.find('div', {'class', 'box box-rounded mt-2'})
+        temp2 = temp.findAll('center')
+        temp = temp2[1].find('i').text
+        if "$" in temp:
+            temp = temp.replace("$", "")
+        USD = cleanString((temp.strip()))
+    except:
+        try:
+            temp = soup.find('div', {'class', 'box box-rounded'})
+            temp2 = temp.findAll('center')
+            temp = temp2[1].find('span').text
+            if "$" in temp:
+                temp = temp.replace("$", "")
+            USD = cleanString((temp.strip()))
+        except:
+            print("USD")
 
     EURO = "-1"  # 14 Product_EURO_SellingPrice
 
 
    # product sold
-    temp = soup.find('div', {'class', 'box rounded mb-0'})  # card sidebar-menu mb-4 card sidebar-menu mb-4
-    temp2 = temp.find('i')
-    temp = temp2.text
-    sold = (cleanString(temp.strip()))
-    # sold = "-1"
+    try:
+        temp = soup.find('div', {'class', 'box rounded mb-0'})  # card sidebar-menu mb-4 card sidebar-menu mb-4
+        temp2 = temp.find('i')
+        temp = temp2.text
+        sold = (cleanString(temp.strip()))
+        # sold = "-1"
+    except:
+        print("product sold")
 
     # product quantatiy left  ###ERRROR
     try:
@@ -180,7 +215,7 @@ def m00nkey_listing_parser(soup):
 
         # vendor
         try:
-            temp = a.find('col-5 justify-content-between mx-auto').find('a').text
+            temp = a.find('div', {'class','col-5 justify-content-between mx-auto'}).find('a').text
             vendor.append(cleanString(temp.strip()))
         except:
             print('vendor')
@@ -188,16 +223,15 @@ def m00nkey_listing_parser(soup):
         #vendor rating
 
 
-        #successful transactions
+        #successful transactions CHECK AGAIN HERE
         try:
-            temp = a.find('col-5 justify-content-between mx-auto').find('div').text
-            success.append(cleanString(temp.strip()))
+            success.append("-1")
         except:
             print('successful transactions')
 
         # product name
         try:
-            temp = a.find('card-title rounded text-truncate').find('a').text
+            temp = a.find('h5', {'class','card-title rounded text-truncate'}).find('a').text
             name.append(cleanString(temp.strip()))
         except:
             print('product name')
@@ -205,27 +239,70 @@ def m00nkey_listing_parser(soup):
 
         CVE.append('-1')
         MS.append('-1')
-        rating_vendor.append("-1")
+        rating_vendor.append('-1')
 
+        # product category
         try:
-            temp = a.findAll('btn btn-block btn-primary')
+            temp = soup.find('div', {'class', 'card-sidebar-menu box mb-2 flex-column'}).find('h3').find('span').text
+            if "Search Results for: " in temp:
+                temp = temp.replace("Search Results for: ", "")
+            category.append(cleanString(temp.strip()))
+
         except:
             print("Error in product category")
 
-    category = []  # 7 Product_Category y
-    describe = []  # 8 Product_Description
-    views = []  # 9 Product_Number_Of_Views
-    reviews = []  # 10 Product_Number_Of_Reviews
-    rating_item = []  # 11 Product_Rating
-    addDate = []  # 12 Product_AddDate
-    BTC = []  # 13 Product_BTC_SellingPrice
-    USD = []  # 14 Product_USD_SellingPrice y
-    EURO = []  # 15 Product_EURO_SellingPrice
-    sold = []  # 16 Product_QuantitySold
-    qLeft = []  # 17 Product_QuantityLeft
-    shipFrom = []  # 18 Product_ShippedFrom
-    shipTo = []  # 19 Product_ShippedTo
-    href = []  # 20 Product_Links
+        describe.append('-1')
+
+        # product views
+        try:
+            temp = a.find('h6',{'class', 'card-subtitle mb-1 text-muted text-truncate'})
+            temp2 = temp.find('i').text
+            views.append(cleanString(temp2.strip()))
+        except:
+            print("Error in views")
+
+        reviews.append('-1')  # 10 Product_Number_Of_Reviews
+        rating_item.append('-1')  # 11 Product_Rating
+        addDate.append('-1')  # 12 Product_AddDate
+
+        # BTC
+        try:
+            temp = a.find('div', {'class', 'col-3 justify-content-between mx-auto'})
+            temp2 = temp.findAll('p')
+            temp = temp2[1].text
+            BTC.append(cleanString(temp.strip()))
+        except:
+            print("BTC")
+
+        #USD ERROR get rid of $
+        try:
+            temp = a.find('div', {'class', 'col-12 justify-content-between mx-auto'}).find('i').text
+            if '$' in temp:
+                temp = temp.replace("$", "")
+            USD.append(cleanString(temp.strip()))  # 14 Product_USD_SellingPrice
+        except:
+            print("USD")
+
+        EURO.append("-1")  # 15 Product_EURO_SellingPrice
+
+        #product sold
+        try:
+            temp = a.find('div', {'class', 'col-12 mx-auto text-truncate text-center flex-fill'}).findAll('p', {'class', 'card-text mb-0'})
+            temp2 = temp[1].find('i').text
+            sold.append(cleanString(temp2.strip()))
+        except:
+            print("product sold")
+
+        qLeft.append('-1')  # 17 Product_QuantityLeft
+        shipFrom.append('-1')  # 18 Product_ShippedFrom
+        shipTo.append('-1')  # 19 Product_ShippedTo
+
+        #href
+        try:
+            temp = a.find('h5', {'class', 'card-title rounded text-truncate'}).find('a').get('href')
+            href.append(temp)  # 20 Product_Links
+        except:
+            print("href")
 
 
     # Populate the final variable (this should be a list with all fields scraped)

setup.ini

@@ -1,11 +1,11 @@
 
 [TOR]
-firefox_binary_path = C:\\Users\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe
-firefox_profile_path = C:\\Users\\Helium\\Desktop\\Tor Browser\\Browser\\TorBrowser\\Data\\Browser\\profile.default
-geckodriver_path = C:\\Users\\Helium\\PycharmProjects\\dw_pipeline_test\\selenium\\geckodriver.exe
+firefox_binary_path = C:\Users\Helium\Desktop\Tor Browser\Browser\firefox.exe
+firefox_profile_path = C:\Users\Helium\Desktop\Tor Browser\Browser\TorBrowser\Data\Browser\profile.default
+geckodriver_path = C:\Users\Helium\PycharmProjects\dw_pipeline_test\selenium\geckodriver.exe
 
 [Project]
-project_directory = C:\\Users\\Helium\\PycharmProjects\\dw_pipeline_test
+project_directory = C:\Users\Helium\PycharmProjects\dw_pipeline_test
 shared_folder = \\VBoxSvr\\Shared
 
 [PostgreSQL]