From 98acc6f12172817040672123d7c920ebd0932674 Mon Sep 17 00:00:00 2001 From: westernmeadow <43891839+westernmeadow@users.noreply.github.com> Date: Wed, 14 Jun 2023 22:24:15 -0700 Subject: [PATCH 1/3] updated markets schema --- Forums/DB_Connection/db_connection.py | 14 -- MarketPlaces/DB_Connection/db_connection.py | 170 ++++++++++-------- MarketPlaces/Initialization/prepare_parser.py | 113 +++++------- MarketPlaces/Tor2door/parser.py | 68 +++---- MarketPlaces/Utilities/utilities.py | 54 +++--- 5 files changed, 196 insertions(+), 223 deletions(-) diff --git a/Forums/DB_Connection/db_connection.py b/Forums/DB_Connection/db_connection.py index 56d578a..619b85e 100644 --- a/Forums/DB_Connection/db_connection.py +++ b/Forums/DB_Connection/db_connection.py @@ -5,20 +5,6 @@ import traceback import time from datetime import date -''' -user -add -forum_id -user_reputation -status_user -interest_user - -post -remove -user_reputation -status_user -interest_user -''' def connectDataBase(): diff --git a/MarketPlaces/DB_Connection/db_connection.py b/MarketPlaces/DB_Connection/db_connection.py index 08b9998..9cabf34 100644 --- a/MarketPlaces/DB_Connection/db_connection.py +++ b/MarketPlaces/DB_Connection/db_connection.py @@ -5,24 +5,6 @@ import traceback import time from datetime import date -''' -vendors -add -mk_id -vendor_rating -successful_transactions - -products -add -product_rating - -remove -date_item -escrow_info -last_viewed -vendor_rating -successful_transaction -''' def connectDataBase(): @@ -40,7 +22,7 @@ def verifyMarketPlace(cur, nameMarket): try: - cur.execute("select id_mk from marketPlaces where name_mk = %(nameMarket)s limit 1", {'nameMarket': nameMarket}) + cur.execute("select market_id from marketPlaces where name_market = %(nameMarket)s limit 1", {'nameMarket': nameMarket}) recset = cur.fetchall() @@ -55,11 +37,12 @@ def verifyMarketPlace(cur, nameMarket): print (trace) -def verifyVendor(cur, nameVendor): +def verifyVendor(cur, nameVendor, marketId): try: - cur.execute("select id_ve from vendors where name_ve = %(nameVendor)s limit 1", {'nameVendor': nameVendor}) + cur.execute("select vendor_id from vendors where name_vendor = %(nameVendor)s and market_id = %(marketId)s " + "limit 1", {'nameVendor': nameVendor, 'marketId': marketId}) recset = cur.fetchall() @@ -78,7 +61,7 @@ def getLastMarketPlace(cur): try: - cur.execute("select id_mk from marketPlaces order by id_mk desc limit 1") + cur.execute("select market_id from marketPlaces order by market_id desc limit 1") recset = cur.fetchall() @@ -97,7 +80,7 @@ def getLastVendor(cur): try: - cur.execute("select id_ve from vendors order by id_ve desc limit 1") + cur.execute("select vendor_id from vendors order by vendor_id desc limit 1") recset = cur.fetchall() @@ -114,54 +97,70 @@ def getLastVendor(cur): def create_marketPlace(cur, row): - marketPlace = verifyMarketPlace(cur, row[0]) + marketId = verifyMarketPlace(cur, row[0]) - if not marketPlace: - marketPlace = int(getLastMarketPlace(cur) + 1) + if not marketId: + marketId = int(getLastMarketPlace(cur) + 1) - sql = "Insert into marketPlaces (id_mk, name_mk, url_mk, dateInserted) Values (%s, %s, %s, %s)" + sql = "Insert into marketplaces (market_id, name_market, url_market, dateinserted_market) " \ + "Values (%s, %s, %s, %s)" - recset = [marketPlace, row[0], None, time.asctime()] + recset = [marketId, row[0], None, row[21]] cur.execute(sql, recset) - return marketPlace + return marketId -def create_vendor(cur, row): +def create_vendor(cur, row, marketId): - vendor = verifyVendor(cur, row[18]) + vendorId = verifyVendor(cur, row[1], marketId) - if not vendor: - vendor = int(getLastVendor(cur) + 1) + if not vendorId: + vendorId = int(getLastVendor(cur) + 1) - sql = "Insert into vendors (id_ve, name_ve, dateInserted) Values (%s, %s, %s)" + sql = "Insert into vendors (vendor_id, market_id, name_vendor, rating_vendor, successfultransactions_vendor, dateinserted_vendor) Values (%s, %s, %s, %s, %s, %s)" - recset = [vendor, row[18], time.asctime()] + recset = [vendorId, marketId, + row[1], + row[2] if row[2] != '-1' else None, + row[3] if row[3] != '-1' else None, + row[21]] cur.execute(sql, recset) - return vendor - - -def create_items(cur, row, marketPlace, vendor): - - sql = "Insert into items (id_mk, id_ve, date_item, name_item, description_item, cve_item, ms_item, category_item, " \ - "escrowinfo_item, views_item, reviews_item, addeddate_item, lastvieweddate_item, btcsellingprice_item, usdsellingprice_item, " \ - "eurosellingprice_item, quantitysold_item, quantityleft_item, shippedfrom_item, shippedto_item, vendorrating, successfulltransactions, " \ - "termsandconditions, dateinserted_item, classification_item) Values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, " \ - "%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)" - - recset = [marketPlace, vendor, str("%02d" %date.today().month) + "/" + str("%02d" %date.today().day) + "/" + str("%04d" %date.today().year), - #recset = [marketPlace, vendor, str("%02d" %date.today().day) + "/" + str("%02d" %date.today().month) + "/" + str("%04d" %date.today().year), - row[1] if row[1]!= '-1' else None, row[5] if row[5]!= '-1' else None, row[2] if row[2]!= '-1' else None, - row[3] if row[3]!= '-1' else None, row[4] if row[4]!= '-1' else None, row[6] if row[6]!= '-1' else None, - row[7] if row[7]!= '-1' else None, row[8] if row[8]!= '-1' else None, row[9] if row[9]!= '-1' else None, - row[10] if row[10]!= '-1' else None, row[11] if row[11]!= '-1' else None, row[12] if row[12]!= '-1' else None, - row[13] if row[13]!= '-1' else None, row[14] if row[14]!= '-1' else None, row[15] if row[15]!= '-1' else None, - row[16] if row[16]!= '-1' else None, row[17] if row[17]!= '-1' else None, row[19] if row[19]!= '-1' else None, - row[20] if row[20]!= '-1' else None, row[21] if row[21]!= '-1' else None, row[22] if row[22]!= '-1' else None, - row[24] if row[24]!= '-1' else None] + return vendorId + + +def create_items(cur, row, marketId, vendorId): + + sql = "Insert into items (market_id, vendor_id, name_item, description_item, cve_item, ms_item, category_item, " \ + "views_item, reviews_item, rating_item, dateadded_item, btc_item, usd_item, euro_item, quantitysold_item, " \ + "quantityleft_item, shippedfrom_item, shippedto_item, href_item, lastseen_item, dateinserted_item, " \ + "classification_item) Values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, " \ + "%s, %s, %s)" + + recset = [marketId, vendorId, + row[4], + row[5] if row[5] != '-1' else None, + row[6] if row[6] != '-1' else None, + row[7] if row[7] != '-1' else None, + row[8] if row[8] != '-1' else None, + row[9] if row[9] != '-1' else None, + row[10] if row[10] != '-1' else None, + row[11] if row[11] != '-1' else None, + row[12] if row[12] != '-1' else None, + row[13] if row[13] != '-1' else None, + row[14] if row[14] != '-1' else None, + row[15] if row[15] != '-1' else None, + row[16] if row[16] != '-1' else None, + row[17] if row[17] != '-1' else None, + row[18] if row[18] != '-1' else None, + row[19] if row[19] != '-1' else None, + row[20] if row[20] != '-1' else None, + row[21], + row[21], + row[22]] cur.execute(sql, recset) @@ -170,26 +169,53 @@ def create_database(cur, con): try: - sql = "create table marketplaces(id_mk integer not null, name_mk character varying(255) not null, " \ - "url_mk character varying(255) null, dateinserted timestamp(6) with time zone not null, " \ - "constraint pk_mk primary key (id_mk))" + sql = "create table marketplaces(market_id integer not null, name_market character varying(255) not null, " \ + "url_market character varying(255) null, dateinserted_market timestamp(6) with time zone not null, " \ + "constraint markets_pk primary key (market_id))" + cur.execute(sql) + + sql = "create table vendors(vendor_id integer not null, market_id integer not null, name_vendor character " \ + "varying(255) not null, rating_vendor character varying(255), successfultransactions_vendor integer " \ + "null, dateinserted_vendor timestamp(6) with time zone not null, constraint vendors_pk primary key (" \ + "vendor_id), constraint vendors_market_id_fkey foreign key (market_id) references marketplaces (" \ + "market_id))" + cur.execute(sql) + + sql = "create table vendors_history(vendor_id integer not null, market_id integer not null, name_vendor " \ + "character varying(255) not null, rating_vendor character varying(255), successfultransactions_vendor " \ + "integer null, dateinserted_vendor timestamp(6) with time zone not null, constraint vendors_history_pk " \ + "primary key (vendor_id, dateinserted_vendor), constraint vendors_history_vendor_id_fkey foreign key (" \ + "vendor_id) references vendors (vendor_id), constraint vendors_history_market_id_fkey foreign key (" \ + "market_id) references marketplaces (market_id))" cur.execute(sql) - sql = "create table vendors(id_ve integer not null, name_ve character varying(255) not null, " \ - "dateinserted timestamp(6) with time zone not null, constraint pk_ve primary key (id_ve))" + sql = "create table items(market_id integer not null, vendor_id integer not null, name_item character " \ + "varying(255) not null, description_item character varying(1000000) null, cve_item character varying(" \ + "255) null, ms_item character varying(255) null, category_item character varying(255) null, views_item " \ + "integer null, reviews_item integer null, rating_item character varying(255) null, dateadded_item " \ + "character varying(25) null, btc_item character varying(255) null, usd_item character varying(255) " \ + "null, euro_item character varying(255) null, quantitysold_item integer null, quantityleft_item " \ + "character varying(255) null, shippedfrom_item character varying(255) null, shippedto_item character " \ + "varying(255) null, href_item character varying(255) null, lastseen_item timestamp(6) with time zone " \ + "not null, dateinserted_item timestamp(6) with time zone not null, classification_item double " \ + "precision not null, constraint items_pk primary key (market_id, vendor_id, name_item), constraint " \ + "items_market_id_fkey foreign key (market_id) references marketplaces (market_id),constraint " \ + "items_vendor_id_fkey foreign key (vendor_id) references vendors (vendor_id))" cur.execute(sql) - sql = "create table items(id_mk integer not null, id_ve integer not null, date_item date not null, name_item character varying(255) not null, " \ - "description_item character varying(1000000) null, cve_item character varying(255) null, ms_item character varying(255) null, " \ - "category_item character varying(255) null, escrowinfo_item character varying(1000) null, views_item integer null, " \ - "reviews_item character varying(255) null, addeddate_item character varying(25) null, " \ - "lastvieweddate_item character varying(25) null, btcsellingprice_item character varying(255) null, " \ - "usdsellingprice_item character varying(255) null, eurosellingprice_item character varying(255) null, quantitysold_item integer null, " \ - "quantityleft_item character varying(255) null, shippedfrom_item character varying(255) null, shippedto_item character varying(5000) null, " \ - "vendorrating character varying(255) null, successfulltransactions character varying(500) null, " \ - "termsandconditions character varying(15000) null, dateinserted_item timestamp(6) with time zone not null, " \ - "classification_item double precision not null, constraint pk_items primary key (id_mk, id_ve, date_item, name_item), " \ - "constraint items_id_mk_fkey foreign key (id_mk) references marketplaces (id_mk),constraint items_id_ve_fkey foreign key (id_ve) references vendors (id_ve))" + sql = "create table items_history(market_id integer not null, vendor_id integer not null, name_item character " \ + "varying(255) not null, description_item character varying(1000000) null, cve_item character varying(" \ + "255) null, ms_item character varying(255) null, category_item character varying(255) null, views_item " \ + "integer null, reviews_item integer null, rating_item character varying(255) null, dateadded_item " \ + "character varying(25) null, btc_item character varying(255) null, usd_item character varying(255) " \ + "null, euro_item character varying(255) null, quantitysold_item integer null, quantityleft_item " \ + "character varying(255) null, shippedfrom_item character varying(255) null, shippedto_item character " \ + "varying(255) null, href_item character varying(255) null, lastseen_item timestamp(6) with time zone " \ + "not null, dateinserted_item timestamp(6) with time zone not null, classification_item double " \ + "precision not null, constraint items_history_pk primary key (market_id, vendor_id, name_item, " \ + "dateinserted_item), constraint items_history_market_id_fkey foreign key (market_id) references " \ + "marketplaces (market_id), constraint items_history_vendor_id_fkey foreign key (vendor_id) references " \ + "vendors (vendor_id))" cur.execute(sql) con.commit() diff --git a/MarketPlaces/Initialization/prepare_parser.py b/MarketPlaces/Initialization/prepare_parser.py index 9ebeabe..5f64321 100644 --- a/MarketPlaces/Initialization/prepare_parser.py +++ b/MarketPlaces/Initialization/prepare_parser.py @@ -18,75 +18,44 @@ def mergePages(rmm, rec): print("----------------- Matched: " + rec[1] + "--------------------") - # if rec[1] == "-1": #Item_Name - # rec[1] = rmm[0] - rec[1] = rmm[0] - - if rec[2] == "-1": #Item_CVE_Classification - rec[2] = rmm[4] - - if rec[3] == "-1": #Item_MS_Classification - rec[3] = rmm[5] - - if rec[4] == "-1": #Item_MarketCategory - rec[4] = rmm[7] - - if rec[5] == "-1": #Item_Description - rec[5] = rmm[1] - elif rmm[1] != "-1": - rec[5] = rec[5] + " " + rmm[1] - - if rec[6] == "-1": #Item _EscrowInfo - rec[6] = rmm[11] - - #rec[7] = "-1" #Item__N.OfViews - - if rec[8] == "-1": #Item_Reviews - rec[8] = rmm[6] - - if rec[9] == "-1": #Item_AddedDate - rec[9] = rmm[15] - - if rec[10] == "-1": #Item_LastViewedDate - rec[10] = rmm[2] - - if rec[11] == "-1": #Item_BTC_SellingPrice - rec[11] = rmm[18] - - if rec[12] == "-1": #Item_US_SellingPrice - rec[12] = rmm[19] - - if rec[13] == "-1": #Item_EURO_SellingPrice - rec[13] = rmm[22] - - if rec[14] == "-1": #Item_QuantitySold - rec[14] = rmm[14] - - if rec[15] == "-1": #Item_QuantityLeft - rec[15] = rmm[10] - - if rec[16] == "-1": #Item_ShippedFrom - rec[16] = rmm[8] - - if rec[17] == "-1": #Item_ShippedTo - rec[17] = rmm[9] - - if rec[18] == "-1": #Vendor_Name - rec[18] = rmm[13] - - if rec[19] == "-1": #Vendor_Rating - rec[19] = rmm[20] - - if rec[20] == "-1": #Vendor_Successfull Transactions - rec[20] = rmm[21] - - if rec[21] == "-1": #Vendor_TermsAndConditions - rec[21] = rmm[12] - - #rec[?] = rmm[17] #Item_EndDate - #rec[?] = rmm[?] #Item_Feedback - #rec[?] = rmm[?] #Shipping Options - #rec[?] = rmm[?] #Average Delivery Time + if rec[1] == "-1": # name_vendor + rec[1] = rmm[0] + if rec[2] == "-1": # rating_vendor + rec[2] = rmm[1] + if rec[3] == "-1": # success_vendor + rec[3] = rmm[2] + if rec[4] == "-1": # name_item + rec[4] = rmm[3] + if rec[5] == "-1": # description_item + rec[5] = rmm[4] + if rec[6] == "-1": # cve_item + rec[6] = rmm[5] + if rec[7] == "-1": # ms_item + rec[7] = rmm[6] + if rec[8] == "-1": # category_item + rec[8] = rmm[7] + if rec[9] == "-1": # views_item + rec[9] = rmm[8] + if rec[10] == "-1": # reviews_item + rec[10] = rmm[9] + if rec[11] == "-1": # rating_item + rec[11] = rmm[10] + if rec[12] == "-1": # adddate_item + rec[12] = rmm[11] + if rec[13] == "-1": # btc_item + rec[13] = rmm[12] + if rec[14] == "-1": # usd_item + rec[14] = rmm[13] + if rec[15] == "-1": # euro_item + rec[15] = rmm[14] + if rec[16] == "-1": # quantitysold_item + rec[16] = rmm[15] + if rec[17] == "-1": # quantityleft_item + rec[17] = rmm[16] + if rec[18] == "-1": # shippedfrom_item + rec[18] = rmm[17] + if rec[19] == "-1": # shippedto_item + rec[19] = rmm[18] return rec @@ -95,7 +64,7 @@ def persist_data(row, cur): marketPlace = create_marketPlace(cur, row) - vendor = create_vendor(cur, row) + vendor = create_vendor(cur, row, marketPlace) create_items(cur, row, marketPlace, vendor) @@ -246,7 +215,7 @@ def new_parse(marketPlace, createLog): # key = u"Pr:" + rec[1].upper()[:list_lim1] + u" Vendor:" + rec[18].upper()[:list_lim2] # key = u"Pr:" + rec[1].upper() - url = ''.join(e for e in rec[23] if e.isalnum()) + url = ''.join(e for e in rec[20] if e.isalnum()) key = u"Url:" + url # if the associated description page is parsed @@ -260,7 +229,7 @@ def new_parse(marketPlace, createLog): # Append to the list the classification of the product # rec.append(str(predict(rec[1], rec[5], language='markets'))) - rec.append(str(predict(rec[1], rec[5], language='sup_english'))) + rec.append(str(predict(rec[4], rec[5], language='sup_english'))) # Persisting the information in the database try: diff --git a/MarketPlaces/Tor2door/parser.py b/MarketPlaces/Tor2door/parser.py index 360d6e7..105fc99 100644 --- a/MarketPlaces/Tor2door/parser.py +++ b/MarketPlaces/Tor2door/parser.py @@ -12,29 +12,25 @@ def tor2door_description_parser(soup): # Fields to be parsed - name = "-1" # 0 Product_Name y - describe = "-1" # 1 Product_Description y - lastSeen = "-1" # 2 Product_LastViewDate - rules = "-1" # 3 NOT USED ... - CVE = "-1" # 4 Product_CVE_Classification (Common Vulnerabilities and Exposures) - MS = "-1" # 5 Product_MS_Classification (Microsoft Security) - review = "-1" # 6 Product_Number_Of_Reviews + vendor = "-1" # 0 Vendor_Name + success = "-1" # 1 Vendor_Successful_Transactions + rating_vendor = "-1" # 2 Vendor_Rating + name = "-1" # 3 Product_Name + describe = "-1" # 4 Product_Description + CVE = "-1" # 5 Product_CVE_Classification (Common Vulnerabilities and Exposures) + MS = "-1" # 6 Product_MS_Classification (Microsoft Security) category = "-1" # 7 Product_Category - shipFrom = "-1" # 8 Product_ShippedFrom - shipTo = "-1" # 9 Product_ShippedTo - left = "-1" # 10 Product_QuantityLeft y - escrow = "-1" # 11 Vendor_Warranty y - terms = "-1" # 12 Vendor_TermsAndConditions - vendor = "-1" # 13 Vendor_Name y - sold = "-1" # 14 Product_QuantitySold y - addDate = "-1" # 15 Product_AddedDate - available = "-1" # 16 NOT USED ... - endDate = "-1" # 17 NOT USED ... - BTC = "-1" # 18 Product_BTC_SellingPrice y - USD = "-1" # 19 Product_USD_SellingPrice y - rating = "-1" # 20 Vendor_Rating - success = "-1" # 21 Vendor_Successful_Transactions - EURO = "-1" # 22 Product_EURO_SellingPrice + views = "-1" # 8 Product_Number_Of_Views + reviews = "-1" # 9 Product_Number_Of_Reviews + rating_item = "-1" # 10 Product_Rating + addDate = "-1" # 11 Product_AddedDate + BTC = "-1" # 12 Product_BTC_SellingPrice + USD = "-1" # 13 Product_USD_SellingPrice + EURO = "-1" # 14 Product_EURO_SellingPrice + sold = "-1" # 15 Product_QuantitySold + left = "-1" # 16 Product_QuantityLeft + shipFrom = "-1" # 17 Product_ShippedFrom + shipTo = "-1" # 18 Product_ShippedTo bae = soup.find('div', {'class': "col-9"}) @@ -57,11 +53,6 @@ def tor2door_description_parser(soup): # half_star = bae[2].find('i', {'class': "fas fa-star-half-alt"}) # rating = len(full_stars) + (0.5 if half_star is not None else 0) - # Finding Warranty - escrow = mb[2].text - escrow = escrow.replace("Payment:", "") - escrow = escrow.strip() - # Finding Quantity Sold and Left temp = mb[4].text.split(',') @@ -116,8 +107,8 @@ def tor2door_description_parser(soup): MS = MS.replace('\n', '') # Populating the final variable (this should be a list with all fields scraped) - row = (name, describe, lastSeen, rules, CVE, MS, review, category, shipFrom, shipTo, left, escrow, terms, vendor, - sold, addDate, available, endDate, BTC, USD, rating, success, EURO) + row = (vendor, rating_vendor, success, name, describe, CVE, MS, category, views, reviews, rating_item, addDate, + BTC, USD, EURO, sold, left, shipFrom, shipTo) # Sending the results return row @@ -128,17 +119,19 @@ def tor2door_listing_parser(soup): # Fields to be parsed nm = 0 # Total_Products (Should be Integer) - mktName = "Tor2door" # 0 Marketplace_Name + mktName = "Tor2door" # 0 Marketplace_Name + vendor = [] # 18 Vendor y + rating_vendor = [] # 19 Vendor_Rating + success = [] # 20 Vendor_Successful_Transactions name = [] # 1 Product_Name y CVE = [] # 2 Product_CVE_Classification (Common Vulnerabilities and Exposures) MS = [] # 3 Product_MS_Classification (Microsoft Security) category = [] # 4 Product_Category y describe = [] # 5 Product_Description - escrow = [] # 6 Vendor_Warranty views = [] # 7 Product_Number_Of_Views - reviews = [] # 8 Product_Number_Of_Reviews y + reviews = [] # 7 Product_Number_Of_Reviews + rating_item = [] # 8 Product_Rating addDate = [] # 9 Product_AddDate - lastSeen = [] # 10 Product_LastViewDate BTC = [] # 11 Product_BTC_SellingPrice USD = [] # 12 Product_USD_SellingPrice y EURO = [] # 13 Product_EURO_SellingPrice @@ -146,10 +139,7 @@ def tor2door_listing_parser(soup): qLeft =[] # 15 Product_QuantityLeft shipFrom = [] # 16 Product_ShippedFrom shipTo = [] # 17 Product_ShippedTo - vendor = [] # 18 Vendor y - rating = [] # 19 Vendor_Rating - success = [] # 20 Vendor_Successful_Transactions - href = [] # 24 Product_Links (Urls) + href = [] # 24 Product_Links listing = soup.findAll('div', {"class": "card product-card mb-3"}) @@ -228,8 +218,8 @@ def tor2door_listing_parser(soup): MS.append(MSValue) # Populate the final variable (this should be a list with all fields scraped) - return organizeProducts(mktName, nm, name, CVE, MS, category, describe, escrow, views, reviews, addDate, lastSeen, - BTC, USD, EURO, qLeft, shipFrom, shipTo, vendor, rating, success, sold, href) + return organizeProducts(mktName, nm, vendor, rating_vendor, success, name, CVE, MS, category, describe, views, + reviews, rating_item, addDate, BTC, USD, EURO, sold, qLeft, shipFrom, shipTo, href) def tor2door_links_parser(soup): diff --git a/MarketPlaces/Utilities/utilities.py b/MarketPlaces/Utilities/utilities.py index 566d612..50bbf89 100644 --- a/MarketPlaces/Utilities/utilities.py +++ b/MarketPlaces/Utilities/utilities.py @@ -207,57 +207,59 @@ def cleanLink(originalLink): return originalLink -def organizeProducts(marketplace, nm, nombre, CVE, MS, category, describe, escrow, views, reviews, addDate, lastSeen, - BTC, USD, EURO, qLeft, shipFrom, shipTo, user, rating, success, sold, href): +def organizeProducts(marketplace, nm, vendor, rating_vendor, success_vendor, nombre, CVE, MS, category, describe, + views, reviews, rating_item, addDate, BTC, USD, EURO, sold, qLeft, shipFrom, shipTo, href): rw = [] day = time.strftime("%m/%d/%Y") - #day = time.strftime("%d/%m/%Y") ahora = time.strftime("%I:%M:%S") for n in range(nm): - lne = marketplace + "," #0 - lne += "-1" if len(nombre) == 0 else nombre[n] #1 + lne = marketplace # 0 + lne += "," + lne += vendor[n] # 1 + lne += "," + lne += "-1" if len(rating_vendor) == 0 else rating_vendor[n] # 2 + lne += "," + lne += "-1" if len(success_vendor) == 0 else success_vendor[n] # 3 + lne += "," + lne += nombre[n] # 4 lne += ',' - lne += "-1" if len(CVE) == 0 else CVE[n] #2 + lne += "-1" if len(describe) == 0 else describe[n] # 5 lne += "," - lne += "-1" if len(MS) == 0 else MS[n] #3 + lne += "-1" if len(CVE) == 0 else CVE[n] # 6 lne += "," - lne += "-1" if len(category) == 0 else category[n] #4 + lne += "-1" if len(MS) == 0 else MS[n] # 7 lne += "," - lne += "-1" if len(describe) == 0 else describe[n] #5 + lne += "-1" if len(category) == 0 else category[n] # 8 lne += "," - lne += "-1" if len(escrow) == 0 else escrow[n] #6 + lne += "-1" if len(views) == 0 else views[n] # 9 lne += "," - lne += "-1" if len(views) == 0 else views[n] #7 + lne += "-1" if len(reviews) == 0 else reviews[n] # 10 lne += "," - lne += "-1" if len(reviews) == 0 else reviews[n] #8 + lne += "-1" if len(rating_item) == 0 else rating_item[n] # 11 lne += "," - lne += "-1" if len(addDate) == 0 else addDate[n] #9 + lne += "-1" if len(addDate) == 0 else addDate[n] # 12 lne += "," - lne += "-1" if len(lastSeen) == 0 else lastSeen[n] #10 + lne += "-1" if len(BTC) == 0 else BTC[n] # 13 lne += "," - lne += "-1" if len(BTC) == 0 else BTC[n] #11 + lne += "-1" if len(USD) == 0 else USD[n] # 14 lne += "," - lne += "-1" if len(USD) == 0 else USD[n] #12 + lne += "-1" if len(EURO) == 0 else EURO[n] # 15 lne += "," - lne += "-1" if len(EURO) == 0 else EURO[n] #13 + lne += "-1" if len(sold) == 0 else sold[n] # 16 lne += "," - lne += "-1" if len(sold) == 0 else sold[n] #14 + lne += "-1" if len(qLeft) == 0 else qLeft[n] # 17 lne += "," - lne += "-1" if len(qLeft) == 0 else qLeft[n] #15 + lne += "-1" if len(shipFrom) == 0 else shipFrom[n] # 18 lne += "," - lne += "-1" if len(shipFrom) == 0 else shipFrom[n] #16 + lne += "-1" if len(shipTo) == 0 else shipTo[n] # 19 lne += "," - lne += "-1" if len(shipTo) == 0 else shipTo[n] #17 - lne += "," + user[n] + "," #18 - lne += "-1" if len(rating) == 0 else rating[n] #19 + lne += "-1" if len(href) == 0 else href[n] # 20 lne += "," - lne += "-1" if len(success) == 0 else success[n] #20 - lne += "," + "-1" + "," + day + " " + ahora + "," #21, 22 - lne += "-1" if len(href) == 0 else href[n] #23 + lne += day + " " + ahora # 21 rw.append(lne) From 921070fa5ee1dc2c6019877fc2ecaeb8e42f3fcc Mon Sep 17 00:00:00 2001 From: Helium Date: Fri, 16 Jun 2023 11:16:41 -0700 Subject: [PATCH 2/3] completed ThifWorld Crawler --- .idea/DW_Pipeline_Test.iml | 2 +- .idea/misc.xml | 2 +- Forums/Altenens/captcha.png | Bin 0 -> 16138 bytes Forums/Altenens/crawler_mechanize.py | 257 +++ Forums/CrackingPro/crawler_selenium.py | 64 +- Forums/CrackingPro/parser.py | 14 +- MarketPlaces/Initialization/marketsList.txt | 2 +- MarketPlaces/Initialization/markets_mining.py | 3 + MarketPlaces/Initialization/prepare_parser.py | 2 +- MarketPlaces/ThiefWorld/crawler_selenium.py | 312 ++++ MarketPlaces/ThiefWorld/geckodriver.log | 1483 +++++++++++++++++ MarketPlaces/ThiefWorld/parser.py | 291 ++++ 12 files changed, 2403 insertions(+), 29 deletions(-) create mode 100644 Forums/Altenens/captcha.png create mode 100644 Forums/Altenens/crawler_mechanize.py create mode 100644 MarketPlaces/ThiefWorld/crawler_selenium.py create mode 100644 MarketPlaces/ThiefWorld/geckodriver.log create mode 100644 MarketPlaces/ThiefWorld/parser.py diff --git a/.idea/DW_Pipeline_Test.iml b/.idea/DW_Pipeline_Test.iml index 7507657..71f5e9b 100644 --- a/.idea/DW_Pipeline_Test.iml +++ b/.idea/DW_Pipeline_Test.iml @@ -2,7 +2,7 @@ - + diff --git a/.idea/misc.xml b/.idea/misc.xml index 730b23f..baf04e9 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -1,4 +1,4 @@ - + \ No newline at end of file diff --git a/Forums/Altenens/captcha.png b/Forums/Altenens/captcha.png new file mode 100644 index 0000000000000000000000000000000000000000..08e45fc90dc94fa4769002bfd3e8107c35e4acfd GIT binary patch literal 16138 zcmW+-19V+o6MeC5+qT&>cGK848{2kct5IV#c4OPN8$18~{`Jl~dAYf7Wt}-QduH!B zH&R(q3JD$`9s~j*$w*760zZd=w;(JuFjMHiBTwI#pGvlNG1cC6)EgTWP~o?9N*p6punT0{-i5JiF!+>VE`3Zv>8VqhexWBF5YD zUUjZw|8(pTUG-e80nuYar$9O0{BaJSRc*%Tj6S$X7*)`SEtQ{Vf>pWSWPu5CCLE{=eK zh{iK6x&xo&|HtQUtZg3dqzdP=bE#Cum4^@*PswArhV z+3mRr@Y3ANl9=3!ttD3Apad3P&ABMpCWw^vTY+yhCSnd{9~=UUuxsie=^-^y)#ffLtC%4`=*TxH#R1Eww*u9(czwF}q>*~i(u`xD`E{3~k9>ACc5w(U zQ=jIPNpd~&a#w3_MXZHiabD%e?2I49zOJ>#QqswQ0ulvhJ2fKb6BUlTSyIeAzrlny zds zx^f)`t=k+N5|4qLTIFp^DRdVC(U6}}ZQ>KEo`DaVV+Z80;XbgiOwBA=NEC!O1xX~N zgg1ElygJr~RCQL%=X2z1%y6UYS$|BKaK~JsVX@$0K?6jBlrrbyCy02YplLV>)OZI7 zkTrIpDGm?7KkO; zQMfOfblnP#yb|H@OBjEt-~0Or2aHEm9*7T?ESdHolCFrMo6HRMOa~dVRN^pVJ$AP= zH+Z)}{iftbODbIduxE)$Y0LAKRENAI)J95TYjM8?pOL_fGuB^l)D|_?znPLTP79{U zQDhUf-Z@>rB0n-7Mj0f;1qFVAeAex!2Lru)H;FY40>{9-%ly`I0}rvQD=tu`ZvCw= zmW~v(O}JRw+!1P2UC=&NSJT^R?;)W{RkL#sEVB)n&g*3uMy?iVM3iQ^3IyWA+7Tu= z$B*x0K6=iO$;HJ}uAiv;=vp1Xl)XEIkob#62QdwiBoN+2{8KONl~1(Mu(5ulwMYbO zZ$>0ag0YuqNlvZc$#ViC0E`_0rUzp74$4qJqNf8@ta!t0?JyM9x^m1mOO|iVgqj>G z_yuRy8U&gytDIN4U}Mn6kZknck(M6jm56X-k!Vj7t^Z>;PrgX%o=q{b*3qSj+{G4{ zpK*0uofn58QU$fSW!b7giwJA#n^kcFEQY_q26uHU4ZlqY_l45<<=hIjj-i((Ylewt zTJ5)HD$mNlq^5YknYJswCgYT3+LHJ6uCg+=gO}4!vY&RWpAaB?@rRC@qR_8r{vM-V zvGAZhbI?U*9Tcg=jcpcK;R5b~6$Xe@9qxO{8)g78Y}K+3vuWiiibV!*>tm_9Ye>P8 zR{1O23i~24W`ZbFr0v86SUy&TSauydM5r|?eArLt!cKuqqB+y{?s=*N~walIt$&wHOXJ4oXLZk_(t4 z)}DDGZODWKF}{ziC*pvzgCZ>Ld)^Se-kk#9ST?NoR8n(9WH>80%B%xD55Kz>KL$5C z+7|V!K1&ZPfwPG_61C@|J`#QLV}E)IAMlS|Op1twptex+8A&|liyuTI(~dSp^1;Q;jvq)9HDJPi1R%~&bn&OJUF>(AEyAziIF+*A zX{d=>(iGX{{sIN;cyuj&nwyqfhX01OEQS23TYUSH0~sJ=P6T*qd!G6tOfMbzE&=N; zT)?E5k}(c&6$SIkV8(rZE+Q0WH?Y>pz%S*Y$ajnxmW5b}IM@eab%t%{<5 z9I#QV#*xuG(b=wEigI`U1xN6Wts{Wf#(HR`setRFV{UXxMjck)rGiHc=4iQF8u|uz>?p1!9iqwB2$otq|$^w?i6D7b^y+Zl6Vg5xEw7cn_x)mwY;$lHzXJ}e5 zLq;~7lG&OgF5IZ(r~zRm%u|LqH~+Ut$gGOhQthD8V#!T}AM^>c3=!s$QNIGl#Z@m9 zS6+`O4hr>Mz^tW+1|#;CiRP8Z6%gOa?J%v?7gPUilW?<@54c{b{2MWkxWsnf!0#v% zcuHJlIp#tG?H9edx*(&xUA9f4IGM^q!n642x`3ghT#yi=2Av_I!r#;GZf_Q;B4Z_1 z$FlzLCT%*RapV)lPXS!A9SR9QuPts%9?Gu3pCXHNI3^;+5|ki-Hg8gtz3bOLKzbzqlIm>t8rr zu8p(KS-coARNIK?q2e0%2*4T)PfK>Hbn14L>JM2~T9!6%a0&cw-izmq7{{^_6vL+~ zW2508GvPx`7k@?#&8zqllUfpfV!a|J-Q(x39B#m(-Ce8)NC>5&NB?bIl}lDL0rZt-i3-D^Ga z`FwwR@x^j^vj0oQJ>^_;1L=6*%vjmAmiSjqYlR=-*7FD;L0R|@6{w99$YgAkJ0Q>& z1MsnITNQNo?Ev6tu3{A}h6S)#Zg{Mw6E#|B#hUA^b{^ zk>9gL>A&YkBCq`8fDc3(NfmPHx$)tdL{M7Wf-d2mpIMEGYEl1pygb#wK#mJxfEG8n zJk9(gVli#TPn}o}6rlXeM<2g0FWpdo|qHrK7TKzlivep zN+GX!<)Uc~1bTWfvM3BJDCFfdO?{nNp8f2&{Z`rwKMbGln=*y<>NIn#y$VKj?v@!D zH}AceIoOHU>h*%>5uYg@xiaM|`6M-r6zf>!O{VhKsU<13I48VYXCr@%JKg#g^bi3t z+lSOOiJn@%f{;i$E#&gVG9-WWjBVvoI-qP9Bxi$IO8uiukE$$iz!_}fL^XUvN*ucU z)_}k7L#a&lICrtkmt3JWYMjJ*Ddn&z0}~)E6^P5#e`5J1CX)z&cXPDt7OB#JuI;!$-7iTn;%Esn5aKE+{uGWISpY_N!Bfn-Wri%4dHjX(Stk zsQKSv9H64(D{ttT7bFKNA|QwMAFvxG#giXwb49eC&VVK9AhLCnZ$3H=S$6wVX_^d0 z&X9I!5mS40KLrm&Xq)%fJ+6v#6$aWK(pp^Tno&SjFhUt5cV< zMO3_+>#U;nj_z|-?67+UZ?Eq?*odzoX$Ur5v2_u*hKr2^0(XN%sZYg!rAT&MhJo&1 z<0R_e&B10e<<$?-u54C=D|DCwL{@MS+~P{ow;aNGHF zE%EkrAg8pm@J&(C8$?gKI;n%oq}!*O^&MxQ5$DqMXl0HNgnHkK%h(7?maLNmGk&aL za_H5)&tJ-8=cOeDJh;otb?d6a^|%Q=!i!fl>XO=N8Een^)%mZ72y1t6jl5#f0e9WS z=}$`uYgrYA{Mhw_7a_&0Q1Sh?`GF|v;Un{4u-+3?;guLM@jz4|_vcU2iN^vvOw`rr zfG#{x?7&)1IW*LgER}yKvZF!N(HCx4FM$K7$&~d*ZhES zH?Q!0@uK}8ftdhrdSj}(5l+|zLf|X5t%R})IPN4Py}yi-Ww9w?vJg?Xz;Ovvt1t)D zVSw3Zk(E)0PS1t=_PWvK+_j>+w2G@1u?mj&XSrf<#6MnbI)Wpc<$YF0PXve%*{ys; z354N}`qb&o*4V>fNA|dSg`(e2&z5m_66T_ua{HSNe8z56;ppzd6iDJ%$>6FL?iq*a zuCo_Je=c8fpCbuL>nYi2XW8*iz`Zk0D}4OS#yG80hD)xO<)r@otJ#d{{usydSW-yC z4Z7jUd#Anh+Z8CL^gEt&wNBGvO0Ll~(-iW%d{>bUAk-R?HO82=7Jbi-j`vbYrqHAg zQ28YPgBwKH97(e?X$(iFP^oWtom9r)Zk}BG`3{Be&4VU3<42BWUah{aD9!@RvIoyx)P21-H+ak zM+^bFxPc7Th5ynXBV9BaIQ`ok2q4tqh~q(UORRR@hTIU&rUprYYgP!xb?s(8Ie8eZ(7Z&$7aqZ{(xLi2|E^j7`Y(2o<&?DbvYG%%fQqajVhal5MU7r&YZtM z#REP`2NMQtK0so+pcW__Esg{>BQpqiHUoZ@>=%{6IjkFezr9!#&nAeg^fXgi|9 z(92ArKC?}HO0bFv{4>xv(5lU>wQry!z6>{%PZ`4{o4+B-DY)^NEn70yS^W&F#z!Ec z5R7oEO(52yOHKKLWiFM6keWCVni%`$5|b3f>HNJ~!|6G#W~jv7N3~R}y7P=$uT`ki z{96Qhq#h=xnSiTJ`q${BEJ~DmZKhN0j!~&BpsL&{Adt0)5N#M)P&P{-&zB%V#I0Zg z>;S+*u`lKGeifGHb^y3)xkWm3dAOR6x=PHE=VKO@&z#c$YJ!NHeiVZ7(ss}?*Av6Z zEvN?sCfzDGW)}q5(3WgUyk~V73u*u#SIjv+0S#*Z+!ZJWRca2kAje=S>8i??&=*Fu zwV8}5ZwXzPSk+AAU9dKGQGgs2R|$3`!R}Kdr1~f|3WZ6-?uAX0h2FvFP+0dP%52u^ zmKOrG$oxkr9LI<%-Ch|J*4kzeMVKdQ+E|F?(9ncTDG@hcy-SbB8~wSm#0@8p7f)t*dvH*gx9opxw?)Yh3PiP*8`0x8t4RF0~dXq5Y~ zIu#t06nyt*F(ExJunh;uKIQoKVRdJ4b;p{t!p3Wj6REi&?WrzH#xpYlnbZXw))o5* z$c=(Bz#IZ^O&;#H53Wtm+Jq0+dfD6i77pt;#?u`D#M7?-hEF+i!djOvO=$uBULkXK z+GPCjt!R>MyNEK zRVkgDAB9G{Xy@>|pz_aEt~_%;6Y0)ZUd6vJeXP4glaYSXU8_J3zjs;WN5Bp2K|<4Y zf-7|mg9^y>OE~7CU}D~)+d@rSbnG#*r>9WI|HvG_WNfvkaDk3f=cOqY$m||TyIT7IOCoqXDytVX+k{nm}1_57kn!%}Nk1_SVFYw>C=Un){w_mwHF1SaR2_EQTa&P8y zUQOWil<0k(t0NpU<@<`@VRoVglE?4ySw&lwk{Ib&J|LiwVrY z*!bZ7aD4m{7Vs8%4F^43-i!sLuU}2H^kNKi_?W0AUu0!v)pvQ_r6>zi-Oz@783`;( zk0$5}3ZYDUXN+UPl<$s3)4A$KtLU=tabdz6WEj!yXr>rm6y#Q$0CUsv~Q1R3d;$3DqQUExObXyd3AqV^>5)z9k%sVklzcOtLB?X^DV zt^loFKS8etXA4h)xcOpDgVaPM1Q1pq2P>i_BB97L|9{}jUfZyj!fjDd-y%{n0MnLD zg$(*4?6ZWL;*+h)t3&Q*V@Tv6L5r>aW@#whjnQ`Uf6-{6f8%~?r24c-Ib zi*+a!E#9G@geIl*%b#$t!h1dd&F6;(RFVl)lD^4cNH=DN*xP_*Htp-g zg4Ypq`StFE>>my%zC?Wmw=mRStyshFr@-g|7)+)dnVO{|DAtx4T9Sq(KQp!{Sz{tr zu_ij+_a0UFt;7atr{#M3@6+Mlmg&$Od(wh6+CRlen{I7SG#F%(0T}yE=gZcv>-X-V zNDdZ?O**F{|8lK;eHh=$mWN{*pWQT;-7hGFMi)=34C`te0WMK*A*SKwRwY<`^>J>- z;{muM?AqWneVvD`IE0q>CLR@cVWD;j?%R-=~%!EcAk@3CMntQ{#!m<8{LxKp2d&(ijOQgkJE%7SjE|{!Uw`pu= znm;ppI{l6b&(7}&CL@P7gdKOxf8Xx#%z6~#|o~byVYLq^zyjbTh^ds;?dux;(?L0EF2rw5d5ON6h$1po+vT2UpoeLM42%MN$lY+x zf5B>tr|897Bf-PH$J##N@-LvnZ}7uJnKArrv{b35(Eh26ciSY5nE)k)fE*1f$>M+< zklxScuT_VDB%So3|7SE8^F7oOQ>EdmoRdV|-gNBM8Vz1bpeSfslfPt+9)u!2C=mFK zZF0==DoYaa881{dJ4327zk>O119r7lZyBa)+avI?xZWtjf~n;l#(f81UQdH#eA`2@TTcQ$PjXAAhAo$uI*#`ne~u?T0e<$cI*VEC_zpVk zgEtrw6xVs)(H9zGMZHxD)S>Y74*iyyLzska*a6B8>D>cRh`xR||QerwQIs2JcOp)PRxBFkP^V0acWT zPZppcl6zCoga$`#V2XJu_ggBP5xKifdM!_|=-sNXVId|&_x)iL)wj2Ed}PZyvT9Y3 z-ao}{6FS$u~9L#ILseNBgjAyL&g$SEFElX)zwEcDSpz8a8``b-1ld*bwzEHB+T zl2)GIOiJ>>*%5keF{ZSYV?w7#5MZM*e$h#C<1jF}x60#=m;t3_BfOa4b%Xyb7Gu}r zT-q7PbAO?f?{?yVo$ctepp+z2=kBRst$I+})~HFAaCt+6+wn7z#N=!s`sps%U=Yp# zJu8&Wt@+&JE!MT8&V0~aDf{n&65&vah<{+O(+q@^qp>G~fR5YGWXOc&=Ae9@hyWMa zRC+FynIv>aM9TWh6jUm06Be%$t_+alssw`6@TkD0s>sy#lY4vG_QebVE~&_5`nBc* zqu;mdO6Bv=i1O)?Z>y?!^e2P3)#Pm6VPYf#xqHG^I()>~;MA6W752Z4{Yu0H_cr0sKVPf+x*rL9NHm3qL;be8ykWOC zgyP1_VwzJxDmBT-N(F%TMex#uRc7vofI|#?pJe9+T8Ea4br2iPJsIk&g0LXYwgLAU zNq<#vxHV)-8d49SYXtPPr$(Xz(iC|e7q+%yxXIQtlI61I>}JCpw#3ixo@RV2h9{nBd{CKaR}b-I+k8JBA|U!{8;M7`4P93g?zt1&To4K#a|8DLg~J;@-{Jbr zJ&(S@b{+B78|avI%is(mG`yz3^Vn(RppYNd))i(dC7y2f!6&*(6iCWQLhVw9KBT^t zQma=bfAKv3qC!}(_>jE3VXN^y4JJcv_^SuuUx9gtpJJ3b-^B)|~WHNu7 zJ9dRdrUazR7ASUl8!?$MXTXFl9Wo3Jkw7s4c7$m&rgKyxMoU_#;UH=LpdPJ@TvHEy zZnjW0Q0}gPLps=w`xt0U23q<7R(%2vX#m$u1eG`mnKK=70sMJ7FAkSf4WFB96l;__ z!DBXf+dIzWaVj?(0mkm|(+RWSF{g)BfgZW6^*iBp;&0$&OEqk(}1ty9=W=J3n1W z_TEyvKklSZ^)c{ppXpTEIeRr?mh1S`q$1S)BeHURp_T~wdY_3&JolG$iT8nOwYMPL z9|>^Lffr~&C;{{{0_6@*%0(VT9gFnr*G=07R~zZy4z(Sl+P#%$Z~pr&kPb+Ck)MQ} z_&5XAy?L;CJ>9Ol6#{Wx{w5z%pnxED|G#+cbl_K};y%cp6~eD34*Y@YXSiTDgZ^C> z_Qa4N=LI^5V7Kcz6%Nzooy;WSO_{P!6l>WFnmfUrBw>_Ycx|?m7}T~qF3rv7dA^%_rV}Xy+~B%jbP{`fA_*A8i68 z(Xvd>&mCXfm%ZhcdI~hy+9+L7-;Z$Yggh+L%EN9b;AdJ_-k%0NM#V@LD3Q_vI;{ zwGdQA)L+$$-(sfb5F}&h$^(`$jSJM%5<@&0G0z~8j zc59m0!y9%aL7XB1_rKd3!`BO`O-|AW_ji1tnP7Ps96=Mgh^s?rX~vC|hLDK>yFh$j z`;n;GPzbeOU?B2~LEdNkL<$7j@;W8!FZ_#R;EVi!Iwq4xCyDwF080n9c5 z%y1B*_nJd*mw^&+o?^zx%gT-q`wL!F~2XLFzETTphPQ z0UY?@j_1}7a3B;Vle(-s;4V;;XV2^BLGDS8OepVtT`%=c`kK!d;(p8nIUsP?U{Z-B z!iYsuL6x=BIv7O&SmDKR_0C|xy4rh7n`NHN_)Pp$q6PQRXiK%b5J2i4L;qw_@Y*O zGDI=Y@y?#VKL>OBEk80yjQjxK{ry}ZETx6yeKfY^DOzJs&$~wC={erW{Q~jQ_jrvz z`D{(ZH{kbS*N8}wgn*WuHj<&s@t5+2-d>vf)jd9?R4{Rbm#__JpwEomAtXJ~>-mb+ zD5H9xJ){YR@G;=iTCn%-z=*SBmssg2c8FzJir7FQ4(%#9fDT;UeQv&6)as2>fpvHQv9?tWWAs;wmVxo%+b58NX9v=4(D1c z%eY?;lPQHm_Pw72P>2@uSIc|5LGOR)bg3ge{`C)Y=Uy^n_UmO9bLhq+IQ|87HETqO z5Q*>zKa$-KtRXE?;x*^f#vu)TemGBdW9>me2wfE%n8yfd#24lJ`L*Mlc-@TQOhw-& zMaSv{=i4(eHz7PQto>+g53m97rwxH$cBDm(s?bt%#Q_csv2mQoVl|KvHq4ZNVz(31 z+{iJ?6KNQGR9;ZDDYHQBV;L#%cz}u86ju+_gqmnu8*_eFn4@%4vD{a-gF^C6u<*N5 ze>;q~4`-)t(x8KhG+pfjGxwW(A$2wb)a-zh+?r89j5zSM+?TV-WeV3o!~4FY1JYK< z14AcjI90Qu?$t!Sd&5P53&je~Z~p zf)nBqIHzzf0Tf&HFJ#{cX;LU;5S$2g6j7@zzNZ|f!OLc6Ti<}e`yu3NA{cOIb@!1A zzyGctk6s_x?Ldux@I)W(kWzPbw8zFnzksF(S8K)>^mal*Ok682&{RFllE`zEqPHNP z=771;cw9+cmp&eh}o=e~D$5``A>{i}T1wHmVPqwnR(wf6kwLn{B@G z`6Q09Wwym0Y`KRqSg^{<6S`*r`4AV<#w83eV++0UOH0K$oZb*Wc1U11+H;xOVoP&I zQA3t2fB>i4%}BR@IX5;!!YiJZp=|LycWB;l@Z9gXCGtP-!2INJMEL0Qz13B40pb|| zB#gmt@AdhsN$?dyVd}1}#1|1rpf1OQ?12C1J7b;?VwB~%;Zh+U_CnFGlqKxmlQKMY zIZ%Dcl&i_dV4QcRSM`{wFQW=nuV9K8Qti z){-jHre9eh_ZLe?eBz@?1V=%2a(LXccG!cONeqYpZRnZAxjN3vmxzWmX^vrT&&s%| zL`NR7?>}DnzEd;7MDqM)?dwsFOexvse_f)*VRz*TDJ`(Q;^s#}*Vu>rUkgFOYQwW} zX#>oIzD>6)V@)pL_0Gp^AW?D?sTSimpI9h|c!Y!5diN?gj(I^|>DL(X3kQOR@Rd&U zUAm_t$ng>lj*h)|3nlWW+&$RtE3*)0FQ`a@R5JbUkZTxeyVQ46w zv}kjf8rQ+aDN{Hak$yxOrs@ZP6_6*td z7H3lRnQ}so`;RdF)K{UgvNC(Q!*_vo-#Nn%(zqlAr}4XH}@kWxE`&x&C-)_)wgh-bBiBZ?yd`v6kskYoFcNsVs*0v-e zo-(0Edt3T|wf+yU_ z-!sO4t)#b|OCN^_t-Z4ngM`mHV$xL_)%|xn%rl``3Lzkm{-`uf5G3L%`p!&oNSV@pUx7cHgLu}ql@v6q5F?YWX@bnm ztG(QHmyV2Xrxp3^Pq=W^eQg!PrF4}UTMQ;zwI8(mU9OrJBf5G_4y*uw0G*<_l2yx? zDZxfOoe>PA8-ya_b!g=C^q{mm%_AKD_;z9DalS)hgsHN0EA+C;rbm9=_XTIK)xv+| zK!sR;R;o|f@5%fy2Inst1XXK>$*+Z;Lve2D%iB_#_5W)-s= zB-ac-#e#DP&R*l$)i61Ec8XtvlaFK%dGYUA7&i1k_x5+w=aacRPW2*#W0SsEhuT;6 zOEXm8qez$^e6iugZ6p*~2&-6+BDYFg(_j1I2)*F|VaI3!&LNBUTXgQXai3x6HQ1qu z({Ve9Pe)s%|D>L=j{`jD-jO{WfOPYIMB)8+EnetB(EDofD6S!t?D$v&VFY+wL0cSa zvK)U--P)R(lI z>c)GK;gU4fJs7V_{TnJf31{K-kqoX&MUb=z4l}_G&&yEi_)wDz5G36^UNAG>Pdu7t zRxm%ehaFwfvSC2EV3*m~!|ER*tDQPOC1H`jr2y>0XgFhs_Q_N7oWvtDAXJ5(v|PRV(p(58j5}QuQ}2!Jx}8H8*t&*ahv~0lJ3mzR27T( zY4Oy`Oy~%DpL3YE&^>Kg$U?pCg_=6(n)-s@UvWGc*9SvLPl6VMgzQ|^jw)>}h(}P) zW8EEb>PcV1-JEW&>42%eM~`|!L{6w-WLr5b9RXH!ywXcAr-)49!tqd#7J+-CuFe+A zH{5(7v#{d5g!G?Ky?1=%!{3D|slo4%vycOtoUuSzn!Sn6QCax)zH1e{PSC9bW6>)G z-FSuGSPNWh8YAF#1$DCPT@duV?E-7Z-(uLq4L{8LW&LchWvbJyB%Gc!PFQ(Ddice; z4Hi3#AOBk(9{U=9`@hyJq~D4@ds2_v(OX>@1|3HTEr*wD+q%LNl6Ii$nBN|EPswfC zRXSIlSwh?2vKwoU`TmYqD1K*>wuRS<_{*gx`L*LvYY=FPIL}#&01Be+$DD=ZcIlL} zqwf8&3+(&|?52D)D%O4Cq*~U#+5HN!na!6!aX%k4=SGo(i?8Hdx=B)!w&DxQBu5() z)MUM>cC{~9XQwWH8IJSs=KhR`R63LthnHFvl&G`_raJj*ozGv1A+rEIyD2|wsUgub zcBFpnI~#{TC+NqVMD}QBsvoLD!JG&K>T8K=tg6((mPUXA#W*_+)PMIdVcxL5o|7jI zrD0tDaOv#s9j_qJ|4TY0vKe=1V2j>u$kSt!uw$rL9*K?WAIqDeH}v*(8q{-{(Oq1m zF3}zF)4UuTe#Df$*Y(j)o#*T0ah-|gU3QQsS&&~ha$U<{lQchkT?;hOL~FHU2ctE= z-Tw49g3Ehb7;AHOvui)^Ez*p_$3f3jchf{z-Ceh_YgoW69`n1CN@*QRS9e2|#aec& zv|8vCVXXMThYFS@R}_twH(o!z!9?_@9%_V&RkZq?>B40rnLQ*EcgmO^!_)2$hpjyy6!ILih%S(pL_9`GGX(4lJk z>m0XVM@t+D@!DWN%Q9&Q1#8tgDNAA;pTLW10hu1XvFDZ6sW=vw3fEO*z8aw^niV|E z1!fl%X%+)pL>V6Unx;wtZC`!LtPME?sjA{$sej4nfWb_rCg=xIF8|5VEb;sRJfs|K zRc_#zomf4InWHJYuetF|T&_zBsybziBvH;_q*-Nc=!R|k22VN-yJO;;f(}yJ5dv7_ z?Y{pvn}l>1F!kAT8>|>E6wFbjd35yXsYoko6sv5`kZ@}hc$j=JN&?#=OM*UC^)zJH zCc?I4P=!K{bkg-V3m)VWiSt%ZRFAYX)OT-;#2`DMMUiebJ1?!;n&yE3Bh?Jxncq&`%y@kDAV=n; zK#HhexPA_4VA3Qd(u%A&OtySfs)hD)xySPn{n2w#X)KN#SEgR8U z&6#W~u$1s)seHbwRy{Ktt}HZGvhFw^6;C6>6U)79N4A2mzzsd5a)33`Y>v+HhI;AktN`suq8elVVsW}}z{B_NIVc_Q~ zB=x=A1Y`p3P9dXY(19~YjjZF`1MjdcJl8K{HGPfV&{HgVJt#`qJt8EpeD)tNRc)=u ztLirMz|$Ar9^T*Wg$yjd?)1>YeFtb+9Z{&n*}iH+)F>nC_#h8RLOy{;{la#FBofR~ z`|k=xkq_EItFqp&CKyii0psFmli-Yl|QP;#^}<3>y9 zY`bL@wG2;rC;oBMw~pKiBnZR&gz5JMr?2j|^bASap#~DIiu~sNN*^Qk;UvaR3OHN> z$z%ol+pFC4&PbnkPZYdQO**YH@1JeYssjR4lsqmG?y|Iy*@U=1T(RK?@f=XCvS94p z3G1b}Xd3Y)*2FBb)Ejd&Rw|*2xDIwZ>!E<7m0=2Bx+xHvvk9UrfNHT|hXtrzhs&8zKg&`);q1~y@R{QfHz zOsQ#jU?+Hp#A5Z!wGPtN5^Pcd?;xiG7oyD695G1w)ItURcxx)hWHyhRPczgKQCdL8 zukm|%aP9Zo;<6sAMbJVB`270Nz!LeYgnhy4A;)jz?f~=bEvcLzbuD3Nkc|~ch&L54 zLdj$c&3C-z{9+xrfjrm)dn_UlQp3R{B2V#-C^zFM|KP|tXN{l-lXQEN6MA44){F1F~11sWDhdh~dNBTdTd5MroGaNRu}o-{gxh?_@Z1BrozC7#M(T zlOTsz!|_+RVF_%~nmB!rF{zgEorqG3#81qiWULoW`F*C4DX6S-@N2R8#BU(J0L}!A zzlYgJgrcF8SX9Y3=3L<}HJR=ryrBGco&eYa5IGWh;bh7^M5M-wPCA(xmu}`fDr#%# z-~rJ7-5G|hR1FUl`viwfnzSdpAQz#>#o|@%h5((*>?%{?4+vvM+njr8XFx#)6s)kf zYi=aziiuvc^MYmr@h=%CJ%Xj}4_S|qmdmIWE^vSPy5#NcG^;rdj(vVBmW7#fM4GXT z3lxJvyzp(S9OP0GzWUg4{3`EAjcXTdnb#}k&r?AJMoNP(0y(H%miRWLFIB6}z>9#R zO;3IFy-=nHE0fgaCXfQ%>*%03^yTCK [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:49847/devtools/browser/8d9a4ca8-bbbf-4b0a-8b35-690690b32551 +1653373125018 Marionette INFO Listening on port 49852 +1653373125297 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1653373206496 Marionette INFO Stopped listening on port 49852 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] + +###!!! [Child][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + +!!! error running onStopped callback: TypeError: callback is not a function + +###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + +1653373206914 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1653374769366 geckodriver INFO Listening on 127.0.0.1:50399 +1653374773780 mozrunner::runner INFO Running command: "C:\\Users\\calsyslab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50400" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileqm14x6" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1653374776823 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:50400/devtools/browser/0f2e3f7b-c209-4cc4-950c-5a0071269a1e +1653374779000 Marionette INFO Listening on port 50405 +1653374779730 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1653374894606 Marionette INFO Stopped listening on port 50405 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +JavaScript error: , line 0: AbortError: Actor 'Conduits' destroyed before query 'RunListener' was resolved +!!! error running onStopped callback: TypeError: callback is not a function + +###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + +1653374895035 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1653502382907 geckodriver INFO Listening on 127.0.0.1:59947 +1653502387411 mozrunner::runner INFO Running command: "C:\\Users\\calsyslab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "59948" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileEzhkbc" +1653502445614 geckodriver INFO Listening on 127.0.0.1:59981 +1653502449949 mozrunner::runner INFO Running command: "C:\\Users\\calsyslab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "59982" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileZ68Lzj" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1653502451354 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:59982/devtools/browser/9aa8a794-4401-49e3-a0da-fbf4f4f73a1b +1653502452911 Marionette INFO Listening on port 59989 +1653502453255 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\calsyslab\AppData\Local\Temp\rust_mozprofileZ68Lzj\thumbnails) because it does not exist +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1653503156100 Marionette INFO Stopped listening on port 59989 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1653503156608 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +ter-change" +1653502591134 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:60018/devtools/browser/6774a36f-ad17-4ef7-9287-55e32b533c33 +1653502593703 Marionette INFO Listening on port 60023 +[Parent 5252, IPC I/O Parent] WARNING: file /var/tmp/build/firefox-190c4b37ac3d/ipc/chromium/src/base/process_util_win.cc:167 +1653502593945 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\calsyslab\AppData\Local\Temp\rust_mozprofileG1qWmb\thumbnails) because it does not exist +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1653502665391 Marionette WARN Ignoring event 'DOMContentLoaded' because document has an invalid readyState of 'complete'. +1653503157264 Marionette INFO Stopped listening on port 60023 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1653503157482 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +ter-change" +1653502771942 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:60052/devtools/browser/809def2d-bc5e-429f-9bcd-0a676607b80c +1653502774472 Marionette INFO Listening on port 60057 +1653502774769 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\calsyslab\AppData\Local\Temp\rust_mozprofileCWM27r\thumbnails) because it does not exist +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1653503158509 Marionette INFO Stopped listening on port 60057 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1653503158688 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +ter-change" +1653503079317 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:60091/devtools/browser/d8c4a07d-de84-4632-9de8-e3f0d8c4ace0 +1653503081923 Marionette INFO Listening on port 60096 +1653503081968 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\calsyslab\AppData\Local\Temp\rust_mozprofileXKX1hs\thumbnails) because it does not exist +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1653503159826 Marionette INFO Stopped listening on port 60096 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function + +###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + +1653503160199 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1653503367305 geckodriver INFO Listening on 127.0.0.1:60124 +1653503371825 mozrunner::runner INFO Running command: "C:\\Users\\calsyslab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "60125" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileX4lMXA" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1653503374146 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:60125/devtools/browser/fa9fcc7b-2131-4bb4-a802-8b5a552c3862 +1653503376323 Marionette INFO Listening on port 60130 +1653503376432 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\calsyslab\AppData\Local\Temp\rust_mozprofileX4lMXA\thumbnails) because it does not exist +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1653503442317 Marionette INFO Stopped listening on port 60130 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +[Parent 948, IPC I/O Parent] WARNING: pipe error: 232: file /var/tmp/build/firefox-190c4b37ac3d/ipc/chromium/src/chrome/common/ipc_channel_win.cc:544 +1653503442697 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1653504443221 geckodriver INFO Listening on 127.0.0.1:60224 +1653504453491 geckodriver INFO Listening on 127.0.0.1:60236 +1653504457996 mozrunner::runner INFO Running command: "C:\\Users\\calsyslab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "60237" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileSPqsKD" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1653504460131 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:60237/devtools/browser/c590b61b-61f7-4512-9398-5a4c487ee846 +1653504462900 Marionette INFO Listening on port 60242 +1653504463215 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1653504560744 Marionette INFO Stopped listening on port 60242 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData] +1653504561296 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1653506682749 geckodriver INFO Listening on 127.0.0.1:60383 +1653506687100 mozrunner::runner INFO Running command: "C:\\Users\\calsyslab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "60384" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile0Mr7uT" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1653506688517 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:60384/devtools/browser/d9f30388-9c6f-4824-a333-59d8b3c60664 +1653506693387 Marionette INFO Listening on port 60389 +1653506693537 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\calsyslab\AppData\Local\Temp\rust_mozprofile0Mr7uT\thumbnails) because it does not exist +1653509248662 Marionette INFO Stopped listening on port 60389 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1653509249360 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +ter-change" +1653508701082 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:60501/devtools/browser/223b5e7f-7605-433d-bdff-198b74c757f4 +1653508704231 Marionette INFO Listening on port 60506 +1653508704344 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1653509175040 Marionette WARN Ignoring event 'pageshow' because document has an invalid readyState of 'interactive'. +1653509547519 Marionette INFO Stopped listening on port 60506 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1653509548207 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1653509560170 geckodriver INFO Listening on 127.0.0.1:60578 +1653509564724 mozrunner::runner INFO Running command: "C:\\Users\\calsyslab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "60579" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileer8tWv" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1653509566143 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:60579/devtools/browser/0b5f209f-0825-45c8-a145-7d02a629f661 +1653509569225 Marionette INFO Listening on port 60584 +1653509569697 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1653510501587 Marionette INFO Stopped listening on port 60584 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1653510502017 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +ter-change" +1653510323934 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:60628/devtools/browser/f5ad0ee7-430b-40aa-a493-357a4ae1d483 +1653510330228 Marionette INFO Listening on port 60633 +1653510330258 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1653510500411 Marionette INFO Stopped listening on port 60633 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1653510500946 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1653510503994 geckodriver INFO Listening on 127.0.0.1:60661 +1653510508636 mozrunner::runner INFO Running command: "C:\\Users\\calsyslab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "60662" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileGoscYO" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1653510512536 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:60662/devtools/browser/4c4771ca-208d-4465-92aa-8843e3d0b349 +1653510514457 Marionette INFO Listening on port 60667 +1653510514695 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1653510660630 Marionette WARN TimedPromise timed out after 500 ms: stacktrace: +TimedPromise/<@chrome://remote/content/marionette/sync.js:238:19 +TimedPromise@chrome://remote/content/marionette/sync.js:223:10 +interaction.flushEventLoop@chrome://remote/content/marionette/interaction.js:431:10 +webdriverClickElement@chrome://remote/content/marionette/interaction.js:179:31 +1653510674337 Marionette WARN TimedPromise timed out after 500 ms: stacktrace: +TimedPromise/<@chrome://remote/content/marionette/sync.js:238:19 +TimedPromise@chrome://remote/content/marionette/sync.js:223:10 +interaction.flushEventLoop@chrome://remote/content/marionette/interaction.js:431:10 +webdriverClickElement@chrome://remote/content/marionette/interaction.js:179:31 +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\calsyslab\AppData\Local\Temp\rust_mozprofileGoscYO\thumbnails) because it does not exist +1653510697080 Marionette WARN TimedPromise timed out after 500 ms: stacktrace: +TimedPromise/<@chrome://remote/content/marionette/sync.js:238:19 +TimedPromise@chrome://remote/content/marionette/sync.js:223:10 +interaction.flushEventLoop@chrome://remote/content/marionette/interaction.js:431:10 +webdriverClickElement@chrome://remote/content/marionette/interaction.js:179:31 +1653510708621 Marionette WARN TimedPromise timed out after 500 ms: stacktrace: +TimedPromise/<@chrome://remote/content/marionette/sync.js:238:19 +TimedPromise@chrome://remote/content/marionette/sync.js:223:10 +interaction.flushEventLoop@chrome://remote/content/marionette/interaction.js:431:10 +webdriverClickElement@chrome://remote/content/marionette/interaction.js:179:31 +1653510717234 Marionette WARN TimedPromise timed out after 500 ms: stacktrace: +TimedPromise/<@chrome://remote/content/marionette/sync.js:238:19 +TimedPromise@chrome://remote/content/marionette/sync.js:223:10 +interaction.flushEventLoop@chrome://remote/content/marionette/interaction.js:431:10 +webdriverClickElement@chrome://remote/content/marionette/interaction.js:179:31 +1653510732934 Marionette WARN TimedPromise timed out after 500 ms: stacktrace: +TimedPromise/<@chrome://remote/content/marionette/sync.js:238:19 +TimedPromise@chrome://remote/content/marionette/sync.js:223:10 +interaction.flushEventLoop@chrome://remote/content/marionette/interaction.js:431:10 +webdriverClickElement@chrome://remote/content/marionette/interaction.js:179:31 +1653510738937 Marionette WARN TimedPromise timed out after 500 ms: stacktrace: +TimedPromise/<@chrome://remote/content/marionette/sync.js:238:19 +TimedPromise@chrome://remote/content/marionette/sync.js:223:10 +interaction.flushEventLoop@chrome://remote/content/marionette/interaction.js:431:10 +webdriverClickElement@chrome://remote/content/marionette/interaction.js:179:31 +1653510744921 Marionette WARN TimedPromise timed out after 500 ms: stacktrace: +TimedPromise/<@chrome://remote/content/marionette/sync.js:238:19 +TimedPromise@chrome://remote/content/marionette/sync.js:223:10 +interaction.flushEventLoop@chrome://remote/content/marionette/interaction.js:431:10 +webdriverClickElement@chrome://remote/content/marionette/interaction.js:179:31 +1653510761608 Marionette WARN TimedPromise timed out after 500 ms: stacktrace: +TimedPromise/<@chrome://remote/content/marionette/sync.js:238:19 +TimedPromise@chrome://remote/content/marionette/sync.js:223:10 +interaction.flushEventLoop@chrome://remote/content/marionette/interaction.js:431:10 +webdriverClickElement@chrome://remote/content/marionette/interaction.js:179:31 +1653510771505 Marionette WARN TimedPromise timed out after 500 ms: stacktrace: +TimedPromise/<@chrome://remote/content/marionette/sync.js:238:19 +TimedPromise@chrome://remote/content/marionette/sync.js:223:10 +interaction.flushEventLoop@chrome://remote/content/marionette/interaction.js:431:10 +webdriverClickElement@chrome://remote/content/marionette/interaction.js:179:31 +1653510783409 Marionette WARN TimedPromise timed out after 500 ms: stacktrace: +TimedPromise/<@chrome://remote/content/marionette/sync.js:238:19 +TimedPromise@chrome://remote/content/marionette/sync.js:223:10 +interaction.flushEventLoop@chrome://remote/content/marionette/interaction.js:431:10 +webdriverClickElement@chrome://remote/content/marionette/interaction.js:179:31 +1653510792536 Marionette WARN TimedPromise timed out after 500 ms: stacktrace: +TimedPromise/<@chrome://remote/content/marionette/sync.js:238:19 +TimedPromise@chrome://remote/content/marionette/sync.js:223:10 +interaction.flushEventLoop@chrome://remote/content/marionette/interaction.js:431:10 +webdriverClickElement@chrome://remote/content/marionette/interaction.js:179:31 +1653510803877 Marionette WARN TimedPromise timed out after 500 ms: stacktrace: +TimedPromise/<@chrome://remote/content/marionette/sync.js:238:19 +TimedPromise@chrome://remote/content/marionette/sync.js:223:10 +interaction.flushEventLoop@chrome://remote/content/marionette/interaction.js:431:10 +webdriverClickElement@chrome://remote/content/marionette/interaction.js:179:31 +1653510816887 Marionette WARN TimedPromise timed out after 500 ms: stacktrace: +TimedPromise/<@chrome://remote/content/marionette/sync.js:238:19 +TimedPromise@chrome://remote/content/marionette/sync.js:223:10 +interaction.flushEventLoop@chrome://remote/content/marionette/interaction.js:431:10 +webdriverClickElement@chrome://remote/content/marionette/interaction.js:179:31 +1653510828216 Marionette WARN TimedPromise timed out after 500 ms: stacktrace: +TimedPromise/<@chrome://remote/content/marionette/sync.js:238:19 +TimedPromise@chrome://remote/content/marionette/sync.js:223:10 +interaction.flushEventLoop@chrome://remote/content/marionette/interaction.js:431:10 +webdriverClickElement@chrome://remote/content/marionette/interaction.js:179:31 +1653510863955 Marionette INFO Stopped listening on port 60667 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1653510864235 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1653510924617 geckodriver INFO Listening on 127.0.0.1:60713 +1653510928999 mozrunner::runner INFO Running command: "C:\\Users\\calsyslab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "60714" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileSydaVM" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1653510930522 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:60714/devtools/browser/3bc951e5-0c4e-42df-acbe-40bc9493bf59 +1653510933827 Marionette INFO Listening on port 60719 +1653510933973 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\calsyslab\AppData\Local\Temp\rust_mozprofileSydaVM\thumbnails) because it does not exist +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1653511963641 Marionette INFO Stopped listening on port 60719 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1653511964187 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1653511968779 geckodriver INFO Listening on 127.0.0.1:60797 +1653511973119 mozrunner::runner INFO Running command: "C:\\Users\\calsyslab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "60798" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilecbPn3O" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1653511974585 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:60798/devtools/browser/69fa661e-fd43-4a0d-aafa-91c3fdf7172e +1653511978274 Marionette INFO Listening on port 60803 +1653511978539 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\calsyslab\AppData\Local\Temp\rust_mozprofilecbPn3O\thumbnails) because it does not exist +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\calsyslab\AppData\Local\Temp\rust_mozprofilecbPn3O\thumbnails) because it does not exist +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\calsyslab\AppData\Local\Temp\rust_mozprofilecbPn3O\thumbnails) because it does not exist +1653519592496 Marionette INFO Stopped listening on port 60803 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function + +###!!! [Parent][MessageChannel] Error: (msgtype=0xB000E,name=PBackgroundIDBDatabase::Msg_CloseAfterInvalidationComplete) Channel error: cannot send/recv + + +###!!! [Parent][MessageChannel] Error: (msgtype=0xB000D,name=PBackgroundIDBDatabase::Msg_Invalidate) Channel error: cannot send/recv + + +###!!! [Parent][MessageChannel] Error: (msgtype=0xB000E,name=PBackgroundIDBDatabase::Msg_CloseAfterInvalidationComplete) Channel error: cannot send/recv + + +###!!! [Parent][MessageChannel] Error: (msgtype=0x140007,name=PBackgroundLSDatabase::Msg_RequestAllowToClose) Channel error: cannot send/recv + + +###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + +[Parent 7232, IPC I/O Parent] WARNING: pipe error: 232: file /var/tmp/build/firefox-190c4b37ac3d/ipc/chromium/src/chrome/common/ipc_channel_win.cc:544 +1653519592872 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655233846366 geckodriver INFO Listening on 127.0.0.1:51522 +1655233924094 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "51523" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileL4LXjy" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655233924765 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:51523/devtools/browser/e28f2c57-fd04-48da-80c1-9afd92897abb +1655233926580 Marionette INFO Listening on port 51530 +1655233926656 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1655234077528 Marionette INFO Stopped listening on port 51530 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +Crash Annotation GraphicsCriticalError: |[C0][GFX1-]: Receive IPC close with reason=AbnormalShutdown (t=1536.34) +###!!! [Child][MessageChannel] Error: (msgtype=0x3900E5,name=PContent::Msg_GraphicsError) Channel closing: too late to send/recv, messages will be lost + +[GFX1-]: Receive IPC close with reason=AbnormalShutdown +1655234078964 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655241030461 geckodriver INFO Listening on 127.0.0.1:49863 +1655241034267 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "49864" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileiOjSJC" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655241034823 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:49864/devtools/browser/fc41b628-f699-4810-9f15-d80bf910c26d +1655241036139 Marionette INFO Listening on port 49871 +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1655241036452 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +JavaScript error: resource:///modules/DoHHeuristics.jsm, line 211: TypeError: gParentalControlsService is undefined +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1655241247360 Marionette INFO Stopped listening on port 49871 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1655241257885 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655244517347 geckodriver INFO Listening on 127.0.0.1:50150 +1655244531664 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50151" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilexazKf5" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655244532716 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:50151/devtools/browser/6d5b0bcb-bac0-45a5-b304-1a0171d7b306 +1655244534825 Marionette INFO Listening on port 50156 +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1655244535462 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://6bb2a1ca-15fd-4844-8c91-bb661acd33cb/593baaae-3678-4868-86ef-57cd5e05fddd'" nsresult: "0x805303f4 ()" location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670" data: yes] +Stack: mainThreadFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15 +mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10 +_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22 +urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17 +_resurrectSource@resource://devtools/server/actors/thread.js:2142:35 +addAllSources@resource://devtools/server/actors/thread.js:1509:14 +watch@resource://devtools/server/actors/resources/sources.js:52:17 +watchResources@resource://devtools/server/actors/resources/index.js:239:19 +_watchTargetResources@resource://devtools/server/actors/targets/target-actor-mixin.js:156:24 +addWatcherDataEntry@resource://devtools/server/actors/targets/target-actor-mixin.js:47:20 +_addWatcherDataEntry@resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm:483:24 +receiveMessage@resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm:425:21 +Line: 670, column: 0 +console.error: ({}) +SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://6bb2a1ca-15fd-4844-8c91-bb661acd33cb/9f9056c0-4c23-4628-bb2d-2fb62a387a1c'" nsresult: "0x805303f4 ()" location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670" data: yes] +Stack: mainThreadFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15 +mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10 +_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22 +urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17 +_resurrectSource@resource://devtools/server/actors/thread.js:2142:35 +addAllSources@resource://devtools/server/actors/thread.js:1509:14 +watch@resource://devtools/server/actors/resources/sources.js:52:17 +watchResources@resource://devtools/server/actors/resources/index.js:239:19 +_watchTargetResources@resource://devtools/server/actors/targets/target-actor-mixin.js:156:24 +addWatcherDataEntry@resource://devtools/server/actors/targets/target-actor-mixin.js:47:20 +_addWatcherDataEntry@resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm:483:24 +receiveMessage@resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm:425:21 +Line: 670, column: 0 +console.error: ({}) +JavaScript error: resource://devtools/shared/DevToolsUtils.js, line 670: Failed to open input source 'blob:moz-extension://6bb2a1ca-15fd-4844-8c91-bb661acd33cb/593baaae-3678-4868-86ef-57cd5e05fddd' +JavaScript error: resource://devtools/shared/DevToolsUtils.js, line 670: Failed to open input source 'blob:moz-extension://6bb2a1ca-15fd-4844-8c91-bb661acd33cb/9f9056c0-4c23-4628-bb2d-2fb62a387a1c' +1655245712794 Marionette INFO Stopped listening on port 50156 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1655245713578 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655245837329 geckodriver INFO Listening on 127.0.0.1:50251 +1655245841463 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50252" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileOnzlK9" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655245842317 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50252/devtools/browser/c30971ef-8468-4903-8c87-2750c733cf21 +1655245844027 Marionette INFO Listening on port 50259 +1655245844403 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +JavaScript error: moz-extension://6bb2a1ca-15fd-4844-8c91-bb661acd33cb/nscl/common/SyncMessage.js, line 230: Error: Could not establish connection. Receiving end does not exist. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: moz-extension://6bb2a1ca-15fd-4844-8c91-bb661acd33cb/nscl/common/log.js, line 36: TypeError: can't access dead object +1655246055066 Marionette INFO Stopped listening on port 50259 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1655246056867 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655248006247 geckodriver INFO Listening on 127.0.0.1:50553 +1655248009991 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50554" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileMRXTMc" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655248011422 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +DevTools listening on ws://localhost:50554/devtools/browser/9ad05dde-f840-4c0a-86cb-654dc1607d3a +1655248013930 Marionette INFO Listening on port 50561 +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1655248014460 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\CALSysLab\AppData\Local\Temp\rust_mozprofileMRXTMc\thumbnails) because it does not exist +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1655248241799 Marionette INFO Stopped listening on port 50561655248253008 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50697" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile8GgRZw" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655248253711 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:50697/devtools/browser/6e616cd3-ecd3-44a6-a9ee-9b97117582b5 +1655248255402 Marionette INFO Listening on port 50708 +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1655248255904 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\CALSysLab\AppData\Local\Temp\rust_mozprofile8GgRZw\thumbnails) because it does not exist +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1655248605407 Marionette INFO Stopped listening on port 50708 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1655248608775 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655248633498 geckodriver INFO Listening on 127.0.0.1:50905 +1655248638228 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50906" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileEe0vCh" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655248638939 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50906/devtools/browser/5803802b-8eff-4a6b-9b37-beff717362bd +1655248642041 Marionette INFO Listening on port 50913 +1655248642102 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +JavaScript error: moz-extension://6bb2a1ca-15fd-4844-8c91-bb661acd33cb/nscl/common/SyncMessage.js, line 230: Error: Could not establish connection. Receiving end does not exist. +JavaScript error: resource:///modules/DoHHeuristics.jsm, line 211: TypeError: gParentalControlsService is undefined +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\CALSysLab\AppData\Local\Temp\rust_mozprofileEe0vCh\thumbnails) because it does not exist +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1655248788758 Marionette WARN Ignoring event 'DOMContentLoaded' because document has an invalid readyState of 'complete'. +JavaScript error: moz-extension://6bb2a1ca-15fd-4844-8c91-bb661acd33cb/nscl/common/log.js, line 36: TypeError: can't access dead object +1655248960306 Marionette INFO Stopped listening on port 50913 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1655248961474 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655249156121 geckodriver INFO Listening on 127.0.0.1:51201 +1655249160500 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "51202" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile8G3G6r" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655249161174 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:51202/devtools/browser/2f6cc650-cd32-4ee8-8715-97f6b0c74251 +1655249163146 Marionette INFO Listening on port 51208 +1655249163269 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +JavaScript error: resource:///modules/DoHHeuristics.jsm, line 211: TypeError: gParentalControlsService is undefined +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\CALSysLab\AppData\Local\Temp\rust_mozprofile8G3G6r\thumbnails) because it does not exist +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1655249211830 Marionette WARN Ignoring event 'DOMContentLoaded' because document has an invalid readyState of 'complete'. +1655249451240 geckodriver INFO Listening on 127.0.0.1:51368 +1655249458745 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "51369" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\TWARNING: At least one completion condition is taking too long to complete. Conditions: [{"name":"ContentParent: id=22abb77b000","state":{"remoteTypePrefix":"web"},"filename":"/var/tmp/build/firefox-6102df3b8d8b/dom/ipc/ContentParent.cpp","lineNumber":3502,"stack":["resource://gre/modules/nsAsyncShutdown.jsm:addBlocker:162","chrome://global/content/elements/browser-custom-element.js:get loadContext:388","chrome://global/content/elements/browser-custom-element.js:construct:989","chrome://global/content/elements/browser-custom-element.js:connectedCallback:326","chrome://browser/content/tabbrowser.js:_setupInitialBrowserAndTab:408","chrome://browser/content/tabbrowser.js:init:50","chrome://browser/content/browser.js:onDOMContentLoaded:1705"]}] Barrier: profile-before-change +1655249468952 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +XULStore.jsm, line 66: Error: Can't find profile directory. +1655249462053 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\CALSysLab\AppData\Local\Temp\rust_mozprofile6EqLH3\thumbnails) because it does not exist +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://6bb2a1ca-15fd-4844-8c91-bb661acd33cb/001721f0-d087-4135-b961-9455c8bab54b'" nsresult: "0x805303f4 ()" location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670" data: yes] +Stack: mainThreadFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15 +mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10 +_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22 +urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17 +_resurrectSource@resource://devtools/server/actors/thread.js:2142:35 +addAllSources@resource://devtools/server/actors/thread.js:1509:14 +watch@resource://devtools/server/actors/resources/sources.js:52:17 +watchResources@resource://devtools/server/actors/resources/index.js:239:19 +_watchTargetResources@resource://devtools/server/actors/targets/target-actor-mixin.js:156:24 +addWatcherDataEntry@resource://devtools/server/actors/targets/target-actor-mixin.js:47:20 +_addWatcherDataEntry@resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm:483:24 +receiveMessage@resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm:425:21 +Line: 670, column: 0 +console.error: ({}) +SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://6bb2a1ca-15fd-4844-8c91-bb661acd33cb/05a07e7b-2b93-4cc2-a99a-cc227fe6cf6d'" nsresult: "0x805303f4 ()" location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670" data: yes] +Stack: mainThreadFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15 +mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10 +_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22 +urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17 +_resurrectSource@resource://devtools/server/actors/thread.js:2142:35 +addAllSources@resource://devtools/server/actors/thread.js:1509:14 +watch@resource://devtools/server/actors/resources/sources.js:52:17 +watchResources@resource://devtools/server/actors/resources/index.js:239:19 +_watchTargetResources@resource://devtools/server/actors/targets/target-actor-mixin.js:156:24 +addWatcherDataEntry@resource://devtools/server/actors/targets/target-actor-mixin.js:47:20 +_addWatcherDataEntry@resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm:483:24 +receiveMessage@resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm:425:21 +Line: 670, column: 0 +console.error: ({}) +JavaScript error: resource://devtools/shared/DevToolsUtils.js, line 670: Failed to open input source 'blob:moz-extension://6bb2a1ca-15fd-4844-8c91-bb661acd33cb/001721f0-d087-4135-b961-9455c8bab54b' +JavaScript error: resource://devtools/shared/DevToolsUtils.js, line 670: Failed to open input source 'blob:moz-extension://6bb2a1ca-15fd-4844-8c91-bb661acd33cb/05a07e7b-2b93-4cc2-a99a-cc227fe6cf6d' +1655249745798 Marionette INFO Stopped listening on port 51385 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function + +###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + +1655249748004 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655249754380 geckodriver INFO Listening on 127.0.0.1:51543 +1655249759129 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "51544" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileeW84Sn" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655249760016 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:51544/devtools/browser/5ada859c-3d28-4d00-8404-94ab7bd098d5 +1655249763016 Marionette INFO Listening on port 51551 +1655249763323 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +JavaScript error: resource:///modules/DoHHeuristics.jsm, line 211: TypeError: gParentalControlsService is undefined +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\CALSysLab\AppData\Local\Temp\rust_mozprofileeW84Sn\thumbnails) because it does not exist +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1655249807907 Marionette WARN Ignoring event 'DOMContentLoaded' because document has an invalid readyState of 'complete'. +1655250065545 Marionette INFO Stopped listening on port 51551 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1655250071925 RemoteAgent ERROR unable to stop listener: [Ex1655250075691 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "51610" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileaoqXJI" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655250076409 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:51610/devtools/browser/93f09bf5-cd01-4300-8e63-dec5724e2ea0 +1655250078309 Marionette INFO Listening on port 51615 +1655250078482 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\CALSysLab\AppData\Local\Temp\rust_mozprofileaoqXJI\thumbnails) because it does not exist +1655250115049 Marionette INFO Stopped listening on port 51615 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData] +1655250116691 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655250275909 geckodriver INFO Listening on 127.0.0.1:51634 +1655250280328 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "51635" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileFo9EgX" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655250280909 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:51635/devtools/browser/16a22ed3-369a-46a6-a757-b302a28bc1b6 +1655250284043 Marionette INFO Listening on port 51640 +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +1655250284211 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\CALSysLab\AppData\Local\Temp\rust_mozprofileFo9EgX\thumbnails) because it does not exist +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1655250590401 Marionette INFO Stopped listening on port 51640 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1655250591725 RemoteAgent ERROR unable to stop listener: [Ex1655250598396 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "51671" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileaYAP63" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655250599166 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:51671/devtools/browser/c0611f3c-decc-4d13-ae35-df7f430d4d37 +1655250601355 Marionette INFO Listening on port 51676 +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +JavaScript error: , line 0: uncaught exception: Object +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1655250601942 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\CALSysLab\AppData\Local\Temp\rust_mozprofileaYAP63\thumbnails) because it does not exist +1655250779014 Marionette INFO Stopped listening on port 51676 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData] +1655250779954 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655250782608 geckodriver INFO Listening on 127.0.0.1:51695 +1655250787013 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "51696" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilecPSsvx" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655250787654 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:51696/devtools/browser/7ad4efa2-c677-433e-9db6-ab4f64aa155b +1655250790816 Marionette INFO Listening on port 51701 +1655250790965 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/ExtensionContent.jsm, line 575: TypeError: PrecompiledScript.executeInGlobal: Argument 1 is not an object. +JavaScript error: resource:///modules/DoHHeuristics.jsm, line 211: TypeError: gParentalControlsService is undefined +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\CALSysLab\AppData\Local\Temp\rust_mozprofilecPSsvx\thumbnails) because it does not exist +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1655760433658 geckodriver INFO Listening on 127.0.0.1:50055 +1655760437906 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50056" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileChh3rq" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655760438358 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50056/devtools/browser/4a2fc47c-b7da-428e-8795-e609889e4812 +1655760439887 Marionette INFO Listening on port 50061 +1655760439961 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655760465682 Marionette INFO Stopped listening on port 50061 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1655760466005 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655760470014 geckodriver INFO Listening on 127.0.0.1:50076 +1655760474267 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50077" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilem9U3qK" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655760474689 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50077/devtools/browser/16e2b6f0-6445-4232-8a09-02f7fc78e9c8 +1655760476104 Marionette INFO Listening on port 50082 +1655760476321 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655760484102 Marionette INFO Stopped listening on port 50082 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function + +###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + +1655760484386 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655762878828 geckodriver INFO Listening on 127.0.0.1:50845 +1655762883203 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50846" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilemMCCRM" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655762883845 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +JavaScript error: resource://gre/modules/ExtensionCommon.jsm, line 2356: Error: listener not re-registered +DevTools listening on ws://localhost:50846/devtools/browser/78d1dd99-2f7c-4257-9445-4c9b55db1433 +1655762886405 Marionette INFO Listening on port 50851 +1655762886454 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655762956672 Marionette INFO Stopped listening on port 50851 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData] +!!! error running onStopped callback: TypeError: callback is not a function +JavaScript error: moz-extension://6bb2a1ca-15fd-4844-8c91-bb661acd33cb/nscl/common/log.js, line 36: TypeError: can't access dead object +1655762957013 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655939535928 geckodriver INFO Listening on 127.0.0.1:51924 +1655939540461 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "51925" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileu7CneN" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655939541063 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:51925/devtools/browser/cc89fd61-e0ae-491a-b7ff-b559130d6f88 +1655939543769 Marionette INFO Listening on port 51957 +1655939544187 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655939580676 Marionette INFO Stopped listening on port 51957 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +[Parent 5732, IPC I/O Parent] WARNING: pipe error: 232: file /var/tmp/build/firefox-6102df3b8d8b/ipc/chromium/src/chrome/common/ipc_channel_win.cc:544 +!!! error running onStopped callback: TypeError: callback is not a function +[Parent 5732, IPC I/O Parent] WARNING: file /var/tmp/build/firefox-6102df3b8d8b/ipc/chromium/src/base/process_util_win.cc:167 +1655939581022 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655939586017 geckodriver INFO Listening on 127.0.0.1:51993 +1655939590396 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "51994" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilehlF9lO" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655939590966 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:51994/devtools/browser/1581196d-dbb9-43a6-aa4f-b0260bc1aca6 +1655939592565 Marionette INFO Listening on port 52008 +1655939593082 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655939601515 Marionette INFO Stopped listening on port 52008 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1655939601826 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655939635644 geckodriver INFO Listening on 127.0.0.1:52056 +1655939640008 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "52057" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileoMeDcn" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655939640469 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:52057/devtools/browser/9ab93f32-9c70-4fcc-9b13-fbdade978c55 +1655939642263 Marionette INFO Listening on port 52062 +1655939642709 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: , line 0: NotFoundError: No such JSWindowActor 'MarionetteEvents' +JavaScript error: , line 0: NotFoundError: No such JSWindowActor 'MarionetteEvents' +1655940230479 Marionette INFO Stopped listening on port 52061655940233127 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "51919" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileVip5O4" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655940233711 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:51919/devtools/browser/d04ea1c5-64e3-4d36-b57d-417566c0fb1b +1655940235295 Marionette INFO Listening on port 51924 +1655940235815 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1655940283486 Marionette WARN Ignoring event 'DOMContentLoaded' because document has an invalid readyState of 'complete'. +1655940596219 Marionette INFO Stopped listening on port 51924 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1655940596492 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655941066670 geckodriver INFO Listening on 127.0.0.1:50482 +1655941071067 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50483" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileE4vgxD" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655941071655 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50483/devtools/browser/69728995-ec7b-4b47-a401-1a8d2e455fa2 +1655941073407 Marionette INFO Listening on port 50488 +1655941073785 RemoteAgent WARN TLS certificate errors will be ignored for this session +1655941086925 Marionette INFO Stopped listening on port 50488 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1655941087247 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1655941116748 geckodriver INFO Listening on 127.0.0.1:50534 +1655941121297 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50535" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile0AxJQ6" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1655941121913 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50535/devtools/browser/0ff2420d-b3bb-4de7-bef9-d9f677e0ec56 +1655941123579 Marionette INFO Listening on port 50540 +1655941123988 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1655941175872 Marionette WARN Ignoring event 'DOMContentLoaded' because document has an invalid readyState of 'complete'. +1655941333271 Marionette INFO Stopped listening on port 50540 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1655941333392 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1656020288946 geckodriver INFO Listening on 127.0.0.1:49904 +1656020292372 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "49905" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile4rtY3m" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1656020293006 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:49905/devtools/browser/2bb49dcd-388a-4bc1-91e8-3110764e33c2 +1656020294545 Marionette INFO Listening on port 49922 +1656020294758 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: , line 0: AbortError: Actor 'TorConnect' destroyed before query 'torconnect:get-init-args' was resolved +1656020298919 Marionette INFO Stopped listening on port 49922 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] + +###!!! [Parent][MessageChannel] Error: (msgtype=0xB000D,name=PBackgroundIDBDatabase::Msg_Invalidate) Channel error: cannot send/recv + + +###!!! [Parent][MessageChannel] Error: (msgtype=0xB000D,name=PBackgroundIDBDatabase::Msg_Invalidate) Channel error: cannot send/recv + + +###!!! [Parent][MessageChannel] Error: (msgtype=0x140007,name=PBackgroundLSDatabase::Msg_RequestAllowToClose) Channel error: cannot send/recv + +!!! error running onStopped callback: TypeError: callback is not a function +[Parent 8124, IPC I/O Parent] WARNING: file /var/tmp/build/firefox-6102df3b8d8b/ipc/chromium/src/base/process_util_win.cc:167 +1656020299333 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1656020313109 geckodriver INFO Listening on 127.0.0.1:49940 +1656020316719 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "49941" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileyOPcu3" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1656020317276 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:49941/devtools/browser/13b2cd46-d43d-4de6-b7b9-510a74f32908 +1656020318624 Marionette INFO Listening on port 49955 +1656020318860 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\CALSysLab\AppData\Local\Temp\rust_mozprofileyOPcu3\thumbnails) because it does not exist +1656020953366 Marionette INFO Stopped listening on port 49955 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +[Parent 3040, IPC I/O Parent] WARNING: file /var/tmp/build/firefox-6102df3b8d8b/ipc/chromium/src/base/process_util_win.cc:167 + +###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + +1656020953469 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1656025629418 geckodriver INFO Listening on 127.0.0.1:50575 +1656025633897 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50576" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileTF8bw8" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1656025634392 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:50576/devtools/browser/8076a2f8-5a16-4eeb-9ccd-cedd5b2a2de9 +1656025635806 Marionette INFO Listening on port 50581 +1656025635959 RemoteAgent WARN TLS certificate errors will be ignored for this session +1656025649033 Marionette INFO Stopped listening on port 50581 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +1656025649299 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1656025675958 geckodriver INFO Listening on 127.0.0.1:50629 +1656025680406 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50630" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile6krFx7" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1656025680829 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50630/devtools/browser/05eaa761-b00a-419a-b901-1131ad321991 +1656025682413 Marionette INFO Listening on port 50635 +1656025682487 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1656025748621 Marionette WARN Ignoring event 'DOMContentLoaded' because document has an invalid readyState of 'complete'. +1656025751208 Marionette INFO Stopped listening on port 50635 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] + +###!!! [Parent][MessageChannel] Error: (msgtype=0xB000E,name=PBackgroundIDBDatabase::Msg_CloseAfterInvalidationComplete) Channel error: cannot send/recv + +JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData] +!!! error running onStopped callback: TypeError: callback is not a function +1656025751532 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1656025909133 geckodriver INFO Listening on 127.0.0.1:50658 +1656025913579 mozrunner::runner INFO Running command: "C:\\Users\\CALSysLab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50659" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilemuVFCY" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: Init" +console.log: "TorConnect: observed profile-after-change" +console.log: "TorConnect: observing topic 'TorBootstrapStatus'" +console.log: "TorConnect: observing topic 'TorBootstrapError'" +console.log: "TorConnect: observing topic 'TorProcessExited'" +console.log: "TorConnect: observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: observing topic 'torsettings:ready'" +console.log: "TorSettings: observed profile-after-change" +1656025914152 Marionette INFO Marionette enabled +console.log: "TorConnect: will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine google@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +DevTools listening on ws://localhost:50659/devtools/browser/96d95bac-eda3-48df-8430-214e303c3857 +1656025915648 Marionette INFO Listening on port 50664 +1656025915780 RemoteAgent WARN TLS certificate errors will be ignored for this session +1656025931381 Marionette INFO Stopped listening on port 50664 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData] +!!! error running onStopped callback: TypeError: callback is not a function +[Parent 2296, IPC I/O Parent] WARNING: file /var/tmp/build/firefox-6102df3b8d8b/ipc/chromium/src/base/process_util_win.cc:167 +1656025931671 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 diff --git a/MarketPlaces/ThiefWorld/parser.py b/MarketPlaces/ThiefWorld/parser.py new file mode 100644 index 0000000..a239075 --- /dev/null +++ b/MarketPlaces/ThiefWorld/parser.py @@ -0,0 +1,291 @@ +__author__ = 'DarkWeb' + +# Here, we are importing the auxiliary functions to clean or convert data +from MarketPlaces.Utilities.utilities import * + +# Here, we are importing BeautifulSoup to search through the HTML tree +from bs4 import BeautifulSoup + + +#parses description pages, so takes html pages of description pages using soup object, and parses it for info it needs +#stores info it needs in different lists, these lists are returned after being organized +#@param: soup object looking at html page of description page +#return: 'row' that contains a variety of lists that each hold info on the description page +def darkfox_description_parser(soup): + + # Fields to be parsed + + name = "-1" # 0 Product_Name + describe = "-1" # 1 Product_Description + lastSeen = "-1" # 2 Product_LastViewDate + rules = "-1" # 3 NOT USED ... + CVE = "-1" # 4 Product_CVE_Classification (Common Vulnerabilities and Exposures) + MS = "-1" # 5 Product_MS_Classification (Microsoft Security) + review = "-1" # 6 Product_Number_Of_Reviews + category = "-1" # 7 Product_Category + shipFrom = "-1" # 8 Product_ShippedFrom + shipTo = "-1" # 9 Product_ShippedTo + left = "-1" # 10 Product_QuantityLeft + escrow = "-1" # 11 Vendor_Warranty + terms = "-1" # 12 Vendor_TermsAndConditions + vendor = "-1" # 13 Vendor_Name + sold = "-1" # 14 Product_QuantitySold + addDate = "-1" # 15 Product_AddedDate + available = "-1" # 16 NOT USED ... + endDate = "-1" # 17 NOT USED ... + BTC = "-1" # 18 Product_BTC_SellingPrice + USD = "-1" # 19 Product_USD_SellingPrice + rating = "-1" # 20 Vendor_Rating + success = "-1" # 21 Vendor_Successful_Transactions + EURO = "-1" # 22 Product_EURO_SellingPrice + + # Finding Product Name + name = soup.find('h1').text + name = name.replace('\n', ' ') + name = name.replace(",", "") + name = name.strip() + + # Finding Vendor + vendor = soup.find('h3').find('a').text.strip() + + # Finding Vendor Rating + rating = soup.find('span', {'class': "tag is-dark"}).text.strip() + + # Finding Successful Transactions + success = soup.find('h3').text + success = success.replace("Vendor: ", "") + success = success.replace(vendor, "") + success = success.replace("(", "") + success = success.replace(")", "") + success = success.strip() + + bae = soup.find('div', {'class': "box"}).find_all('ul') + + # Finding Prices + USD = bae[1].find('strong').text.strip() + + li = bae[2].find_all('li') + + # Finding Escrow + escrow = li[0].find('span', {'class': "tag is-dark"}).text.strip() + + # Finding the Product Category + category = li[1].find('span', {'class': "tag is-dark"}).text.strip() + + # Finding the Product Quantity Available + left = li[3].find('span', {'class': "tag is-dark"}).text.strip() + + # Finding Number Sold + sold = li[4].find('span', {'class': "tag is-dark"}).text.strip() + + li = bae[3].find_all('li') + + # Finding Shipment Information (Origin) + if "Ships from:" in li[-2].text: + shipFrom = li[-2].text + shipFrom = shipFrom.replace("Ships from: ", "") + # shipFrom = shipFrom.replace(",", "") + shipFrom = shipFrom.strip() + + # Finding Shipment Information (Destination) + shipTo = li[-1].find('div', {'title': "List of countries is scrollable"}).text + shipTo = shipTo.replace("Ships to: ", "") + shipTo = shipTo.strip() + if "certain countries" in shipTo: + countries = "" + tags = li[-1].find_all('span', {'class': "tag"}) + for tag in tags: + country = tag.text.strip() + countries += country + ", " + shipTo = countries.strip(", ") + + # Finding the Product description + describe = soup.find('div', {'class': "pre-line"}).text + describe = describe.replace("\n", " ") + describe = describe.strip() + + '''# Finding the Number of Product Reviews + tag = soup.findAll(text=re.compile('Reviews')) + for index in tag: + reviews = index + par = reviews.find('(') + if par >=0: + reviews = reviews.replace("Reviews (","") + reviews = reviews.replace(")","") + reviews = reviews.split(",") + review = str(abs(int(reviews[0])) + abs(int(reviews[1]))) + else : + review = "-1"''' + + # Searching for CVE and MS categories + cve = soup.findAll(text=re.compile('CVE-\d{4}-\d{4}')) + if cve: + CVE = " " + for idx in cve: + CVE += (idx) + CVE += " " + CVE = CVE.replace(',', ' ') + CVE = CVE.replace('\n', '') + ms = soup.findAll(text=re.compile('MS\d{2}-\d{3}')) + if ms: + MS = " " + for im in ms: + MS += (im) + MS += " " + MS = MS.replace(',', ' ') + MS = MS.replace('\n', '') + + # Populating the final variable (this should be a list with all fields scraped) + row = (name, describe, lastSeen, rules, CVE, MS, review, category, shipFrom, shipTo, left, escrow, terms, vendor, + sold, addDate, available, endDate, BTC, USD, rating, success, EURO) + + # Sending the results + return row + + +#parses listing pages, so takes html pages of listing pages using soup object, and parses it for info it needs +#stores info it needs in different lists, these lists are returned after being organized +#@param: soup object looking at html page of listing page +#return: 'row' that contains a variety of lists that each hold info on the listing page +def darkfox_listing_parser(soup): + + # Fields to be parsed + nm = 0 # Total_Products (Should be Integer) + mktName = "DarkFox" # 0 Marketplace_Name + name = [] # 1 Product_Name + CVE = [] # 2 Product_CVE_Classification (Common Vulnerabilities and Exposures) + MS = [] # 3 Product_MS_Classification (Microsoft Security) + category = [] # 4 Product_Category + describe = [] # 5 Product_Description + escrow = [] # 6 Vendor_Warranty + views = [] # 7 Product_Number_Of_Views + reviews = [] # 8 Product_Number_Of_Reviews + addDate = [] # 9 Product_AddDate + lastSeen = [] # 10 Product_LastViewDate + BTC = [] # 11 Product_BTC_SellingPrice + USD = [] # 12 Product_USD_SellingPrice + EURO = [] # 13 Product_EURO_SellingPrice + sold = [] # 14 Product_QuantitySold + qLeft =[] # 15 Product_QuantityLeft + shipFrom = [] # 16 Product_ShippedFrom + shipTo = [] # 17 Product_ShippedTo + vendor = [] # 18 Vendor + rating = [] # 19 Vendor_Rating + success = [] # 20 Vendor_Successful_Transactions + href = [] # 23 Product_Links (Urls) + + listing = soup.findAll('div', {"class": "card"}) + + # Populating the Number of Products + nm = len(listing) + + for a in listing: + bae = a.findAll('a', href=True) + + # Adding the url to the list of urls + link = bae[0].get('href') + link = cleanLink(link) + href.append(link) + + # Finding the Product + product = bae[1].find('p').text + product = product.replace('\n', ' ') + product = product.replace(",", "") + product = product.replace("...", "") + product = product.strip() + name.append(product) + + bae = a.find('div', {'class': "media-content"}).find('div').find_all('div') + + if len(bae) >= 5: + # Finding Prices + price = bae[0].text + ud = price.replace(" USD", " ") + # u = ud.replace("$","") + u = ud.replace(",", "") + u = u.strip() + USD.append(u) + # bc = (prc[1]).strip(' BTC') + # BTC.append(bc) + + # Finding the Vendor + vendor_name = bae[1].find('a').text + vendor_name = vendor_name.replace(",", "") + vendor_name = vendor_name.strip() + vendor.append(vendor_name) + + # Finding the Category + cat = bae[2].find('small').text + cat = cat.replace("Category: ", "") + cat = cat.replace(",", "") + cat = cat.strip() + category.append(cat) + + # Finding Number Sold and Quantity Left + num = bae[3].text + num = num.replace("Sold: ", "") + num = num.strip() + sold.append(num) + + quant = bae[4].find('small').text + quant = quant.replace("In stock: ", "") + quant = quant.strip() + qLeft.append(quant) + + # Finding Successful Transactions + freq = bae[1].text + freq = freq.replace(vendor_name, "") + freq = re.sub(r'Vendor Level \d+', "", freq) + freq = freq.replace("(", "") + freq = freq.replace(")", "") + freq = freq.strip() + success.append(freq) + + # Searching for CVE and MS categories + cve = a.findAll(text=re.compile('CVE-\d{4}-\d{4}')) + if not cve: + cveValue="-1" + else: + cee = " " + for idx in cve: + cee += (idx) + cee += " " + cee = cee.replace(',', ' ') + cee = cee.replace('\n', '') + cveValue=cee + CVE.append(cveValue) + + ms = a.findAll(text=re.compile('MS\d{2}-\d{3}')) + if not ms: + MSValue="-1" + else: + me = " " + for im in ms: + me += (im) + me += " " + me = me.replace(',', ' ') + me = me.replace('\n', '') + MSValue=me + MS.append(MSValue) + + # Populate the final variable (this should be a list with all fields scraped) + return organizeProducts(mktName, nm, name, CVE, MS, category, describe, escrow, views, reviews, addDate, lastSeen, + BTC, USD, EURO, qLeft, shipFrom, shipTo, vendor, rating, success, sold, href) + + +#called by the crawler to get description links on a listing page +#@param: beautifulsoup object that is using the correct html page (listing page) +#return: list of description links from a listing page +def thiefworld_links_parser(soup): + + # Returning all links that should be visited by the Crawler + + href = [] + listing = soup.find('div', {"class": "row tile__list tileitems_filter pad15 tileproduct__list"}).findAll('div', {"class": "desc"}) + + for a in listing: + bae = a.find('div', {"class": "title"}).find('a', href=True) + link = bae['href'] + href.append(link) + + return href \ No newline at end of file From d30c8066e307536b5e951ec07a15f08833074d5e Mon Sep 17 00:00:00 2001 From: westernmeadow Date: Tue, 20 Jun 2023 02:14:23 -0700 Subject: [PATCH 3/3] added setup.ini, global date, and persisting urls --- .idea/DW_Pipeline_Test.iml | 4 +- .idea/misc.xml | 2 +- Forums/CryptBB/crawler_selenium.py | 66 +++++----- Forums/CryptBB/parser.py | 57 ++++----- Forums/DB_Connection/db_connection.py | 16 ++- Forums/Initialization/forums_mining.py | 17 ++- Forums/Initialization/geckodriver.log | 119 ++++++++++++++++++ Forums/Initialization/prepare_parser.py | 41 +++--- Forums/Utilities/utilities.py | 2 +- MarketPlaces/DB_Connection/db_connection.py | 16 ++- MarketPlaces/Initialization/geckodriver.log | 32 +++++ MarketPlaces/Initialization/marketsList.txt | 2 +- MarketPlaces/Initialization/markets_mining.py | 6 +- MarketPlaces/Initialization/prepare_parser.py | 41 +++--- MarketPlaces/ThiefWorld/crawler_selenium.py | 55 ++++---- MarketPlaces/Tor2door/crawler_selenium.py | 55 ++++---- MarketPlaces/Tor2door/parser.py | 49 ++++---- path.txt | 3 - setup.ini | 14 +++ 19 files changed, 370 insertions(+), 227 deletions(-) delete mode 100644 path.txt create mode 100644 setup.ini diff --git a/.idea/DW_Pipeline_Test.iml b/.idea/DW_Pipeline_Test.iml index 71f5e9b..11bc817 100644 --- a/.idea/DW_Pipeline_Test.iml +++ b/.idea/DW_Pipeline_Test.iml @@ -2,7 +2,7 @@ - + @@ -12,6 +12,8 @@ diff --git a/.idea/misc.xml b/.idea/misc.xml index baf04e9..61a3499 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -1,4 +1,4 @@ - + \ No newline at end of file diff --git a/Forums/CryptBB/crawler_selenium.py b/Forums/CryptBB/crawler_selenium.py index 44db724..655f39a 100644 --- a/Forums/CryptBB/crawler_selenium.py +++ b/Forums/CryptBB/crawler_selenium.py @@ -12,17 +12,19 @@ from selenium.webdriver.firefox.service import Service from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.ui import WebDriverWait -from PIL import Image +from PIL import Image import urllib.parse as urlparse import os, re, time -from datetime import date import subprocess +import configparser from bs4 import BeautifulSoup from Forums.Initialization.prepare_parser import new_parse from Forums.CryptBB.parser import cryptBB_links_parser from Forums.Utilities.utilities import cleanHTML +config = configparser.ConfigParser() +config.read('../../setup.ini') counter = 1 baseURL = 'http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/' @@ -41,15 +43,14 @@ def startCrawling(): print(driver.current_url, e) closetor(driver) - # new_parse(forumName, False) + # new_parse(forumName, baseURL, False) # Opens Tor Browser def opentor(): global pid print("Connecting Tor...") - path = open('../../path.txt').readline().strip() - pro = subprocess.Popen(path) + pro = subprocess.Popen(config.get('TOR', 'firefox_binary_path')) pid = pro.pid time.sleep(7.5) input('Tor Connected. Press ENTER to continue\n') @@ -132,12 +133,9 @@ def closetor(driver): # Creates FireFox 'driver' and configure its 'Profile' # to use Tor proxy and socket def createFFDriver(): - file = open('../../path.txt', 'r') - lines = file.readlines() - - ff_binary = FirefoxBinary(lines[0].strip()) + ff_binary = FirefoxBinary(config.get('TOR', 'firefox_binary_path')) - ff_prof = FirefoxProfile(lines[1].strip()) + ff_prof = FirefoxProfile(config.get('TOR', 'firefox_profile_path')) ff_prof.set_preference("places.history.enabled", False) ff_prof.set_preference("privacy.clearOnShutdown.offlineApps", True) ff_prof.set_preference("privacy.clearOnShutdown.passwords", True) @@ -145,7 +143,7 @@ def createFFDriver(): ff_prof.set_preference("privacy.sanitize.sanitizeOnShutdown", True) ff_prof.set_preference("signon.rememberSignons", False) ff_prof.set_preference("network.cookie.lifetimePolicy", 2) - ff_prof.set_preference("network.dns.disablePrefetch", True)# + ff_prof.set_preference("network.dns.disablePrefetch", True) ff_prof.set_preference("network.http.sendRefererHeader", 0) ff_prof.set_preference("permissions.default.image", 3) ff_prof.set_preference("browser.download.folderList", 2) @@ -159,7 +157,7 @@ def createFFDriver(): ff_prof.set_preference("javascript.enabled", True) ff_prof.update_preferences() - service = Service(lines[2].strip()) + service = Service(config.get('TOR', 'geckodriver_path')) driver = webdriver.Firefox(firefox_binary=ff_binary, firefox_profile=ff_prof, service=service) @@ -170,10 +168,10 @@ def getAccess(): url = getFixedURL() driver = createFFDriver() try: - driver.get(url)# open url in browser + driver.get(url) return driver except: - driver.close()# close tab + driver.close() return 'down' @@ -188,15 +186,12 @@ def savePage(page, url): # Gets the full path of the page to be saved along with its appropriate file name def getFullPathName(url): + from Forums.Initialization.forums_mining import CURRENT_DATE fileName = getNameFromURL(url) if isDescriptionLink(url): - fullPath = r'..\\CryptBB\\HTML_Pages\\' + str( - "%02d" % date.today().month) + str("%02d" % date.today().day) + str( - "%04d" % date.today().year) + r'\\' + r'Description\\' + fileName + '.html' + fullPath = r'..\\CryptBB\\HTML_Pages\\' + CURRENT_DATE + r'\\Description\\' + fileName + '.html' else: - fullPath = r'..\\CryptBB\\HTML_Pages\\' + str( - "%02d" % date.today().month) + str("%02d" % date.today().day) + str( - "%04d" % date.today().year) + r'\\' + r'Listing\\' + fileName + '.html' + fullPath = r'..\\CryptBB\\HTML_Pages\\' + CURRENT_DATE + r'\\Listing\\' + fileName + '.html' return fullPath @@ -204,7 +199,7 @@ def getFullPathName(url): def getNameFromURL(url): global counter name = ''.join(e for e in url if e.isalnum()) - if (name == ''): + if name == '': name = str(counter) counter = counter + 1 return name @@ -226,7 +221,7 @@ def getInterestedLinks(): # # Training Challenges # links.append('http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/forumdisplay.php?fid=96') # Darknet Discussions - #links.append('http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/forumdisplay.php?fid=88') + # links.append('http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/forumdisplay.php?fid=88') # # Public Leaks and Warez # links.append('http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/forumdisplay.php?fid=97') # # Hacked Accounts and Database Dumps @@ -251,7 +246,7 @@ def crawlForum(driver): print('Crawling :', link) try: try: - driver.get(link)# open + driver.get(link) except: driver.refresh() html = driver.page_source @@ -259,10 +254,17 @@ def crawlForum(driver): has_next_page = True - #loop through the topics while has_next_page: - list = topicPages(html)# for multiple pages + list = topicPages(html) for item in list: + itemURL = urlparse.urljoin(baseURL, str(item)) + try: + driver.get(itemURL) + except: + driver.refresh() + savePage(driver.page_source, item) + driver.back() + ''' #variable to check if there is a next page for the topic has_next_topic_page = True counter = 1 @@ -291,18 +293,19 @@ def crawlForum(driver): except NoSuchElementException: has_next_topic_page = False - #end of loop + # end of loop for i in range(counter): driver.back() + ''' # comment out - #break + break # comment out - #if count == 1: - # count = 0 - # break + if count == 1: + count = 0 + break - try:# change depending on web page, #next page + try: temp = driver.find_element(by=By.XPATH, value = '/html/body/div/div[2]/div/div[2]/div') link = temp.find_element(by=By.CLASS_NAME, value='pagination_next').get_attribute('href') @@ -346,7 +349,6 @@ def isListingLink(url): # calling the parser to define the links def topicPages(html): soup = BeautifulSoup(html, "html.parser") - #print(soup.find('div', id="container").find('div', id="content").find('table', {"class": "tborder clear"}).find('tbody').find('tr',{"class": "inline_row"}).find('strong').text) return cryptBB_links_parser(soup) diff --git a/Forums/CryptBB/parser.py b/Forums/CryptBB/parser.py index 0957b76..318b04e 100644 --- a/Forums/CryptBB/parser.py +++ b/Forums/CryptBB/parser.py @@ -15,15 +15,15 @@ def cryptBB_description_parser(soup): # Fields to be parsed - topic = "-1" # topic name - user = [] # all users of each post - addDate = [] # all dated of each post - feedback = [] # all feedbacks of each vendor (this was found in just one Forum and with a number format) - status = [] # all user's authority in each post such as (adm, member, dangerous) - reputation = [] # all user's karma in each post (usually found as a number) - sign = [] # all user's signature in each post (usually a standard message after the content of the post) - post = [] # all messages of each post - interest = [] # all user's interest in each post + topic = "-1" # 0 *topic name + user = [] # 1 *all users of each post + status = [] # 2 all user's authority in each post such as (adm, member, dangerous) + reputation = [] # 3 all user's karma in each post (usually found as a number) + interest = [] # 4 all user's interest in each post + sign = [] # 5 all user's signature in each post (usually a standard message after the content of the post) + post = [] # 6 all messages of each post + feedback = [] # 7 all feedbacks of each vendor (this was found in just one Forum and with a number format) + addDate = [] # 8 all dated of each post # Finding the topic (should be just one coming from the Listing Page) @@ -154,20 +154,6 @@ def cryptBB_description_parser(soup): feedback.append("-1") - ''' - except: - if soup.find('td', {"class": "trow1"}).text == " You do not have permission to access this page. ": - user.append("-1") - status.append(-1) - interest.append(-1) - reputation.append(-1) - addDate.append(-1) - post.append("NO ACCESS TO THIS PAGE!") - sign.append(-1) - feedback.append(-1) - ''' - - # Populate the final variable (this should be a list with all fields scraped) row = (topic, user, status, reputation, interest, sign, post, feedback, addDate) @@ -180,17 +166,17 @@ def cryptBB_description_parser(soup): def cryptBB_listing_parser(soup): - board = "-1" # board name (the previous level of the topic in the Forum categorization tree. - # For instance: Security/Malware/Tools to hack Facebook. The board here should be Malware) - - nm = 0 # this variable should receive the number of topics - topic = [] # all topics - author = [] # all authors of each topic - views = [] # number of views of each topic - posts = [] # number of posts of each topic - addDate = [] # when the topic was created (difficult to find) - href = [] # this variable should receive all cleaned urls (we will use this to do the marge between - # Listing and Description pages) + nm = 0 # *this variable should receive the number of topics + forum = "CryptBB" # 0 *forum name + board = "-1" # 1 *board name (the previous level of the topic in the Forum categorization tree. + # For instance: Security/Malware/Tools to hack Facebook. The board here should be Malware) + topic = [] # 2 *all topics + author = [] # 3 *all authors of each topic + views = [] # 4 number of views of each topic + posts = [] # 5 number of posts of each topic + href = [] # 6 this variable should receive all cleaned urls (we will use this to do the marge between + # Listing and Description pages) + addDate = [] # 7 when the topic was created (difficult to find) # Finding the board (should be just one) @@ -223,7 +209,6 @@ def cryptBB_listing_parser(soup): link = itopic.find('span', {"class": "subject_old"}).find('a').get('href') except: link = itopic.find('span',{"class": "subject_new"}).find('a').get('href') - link = cleanLink(link) href.append(link) # Finding the author of the topic @@ -245,7 +230,7 @@ def cryptBB_listing_parser(soup): addDate.append("-1") - return organizeTopics("CryptBB", nm, topic, board, author, views, posts, href, addDate) + return organizeTopics(forum, nm, board, author, topic, views, posts, href, addDate) def cryptBB_links_parser(soup): diff --git a/Forums/DB_Connection/db_connection.py b/Forums/DB_Connection/db_connection.py index 619b85e..eeaf69b 100644 --- a/Forums/DB_Connection/db_connection.py +++ b/Forums/DB_Connection/db_connection.py @@ -2,15 +2,21 @@ __author__ = 'DarkWeb' import psycopg2 import traceback -import time -from datetime import date +import configparser def connectDataBase(): try: - return psycopg2.connect(host='localhost', user='postgres', password='password', dbname='darkweb_markets_forums') + config = configparser.ConfigParser() + config.read('../../setup.ini') + ip = config.get('PostgreSQL', 'ip') + username = config.get('PostgreSQL', 'username') + password = config.get('PostgreSQL', 'password') + database = config.get('PostgreSQL', 'database') + + return psycopg2.connect(host=ip, user=username, password=password, dbname=database) except: @@ -197,7 +203,7 @@ def getLastPost(cur): ''' -def create_forum(cur, row): +def create_forum(cur, row, url): forumId = verifyForum(cur, row[0]) @@ -207,7 +213,7 @@ def create_forum(cur, row): sql = "Insert into forums (forum_id, name_forum, url_forum, dateinserted_forum) Values (%s, %s, %s, %s)" - recset = [forumId, row[0], None, row[8]] + recset = [forumId, row[0], url, row[8]] cur.execute(sql, recset) diff --git a/Forums/Initialization/forums_mining.py b/Forums/Initialization/forums_mining.py index 71907e0..f431f97 100644 --- a/Forums/Initialization/forums_mining.py +++ b/Forums/Initialization/forums_mining.py @@ -9,10 +9,12 @@ from datetime import * from Forums.BestCardingWorld.crawler_selenium import crawler as crawlerBestCardingWorld from Forums.CryptBB.crawler_selenium import crawler as crawlerCryptBB from Forums.OnniForums.crawler_selenium import crawler as crawlerOnniForums -#from Forums.CrackingPro.crawler_selenium import crawler as crawlerCrackingPro +# from Forums.CrackingPro.crawler_selenium import crawler as crawlerCrackingPro import time +CURRENT_DATE = str("%02d" % date.today().month) + str("%02d" % date.today().day) + str("%04d" % date.today().year) + # reads list of marketplaces manually inputted def getForums(): @@ -30,8 +32,6 @@ def createDirectory(forum): pagesMainDir = '../' + forum else: pagesMainDir = '../' + forum + "/HTML_Pages" - # sharedFolderPath = r'\\VBoxSvr\VM_Files_(shared)' - # pagesMainDir = os.path.join(sharedFolderPath, 'HTML/Forums/' + forum + '/HTML_Pages') if not os.path.isdir(pagesMainDir): os.makedirs(pagesMainDir) @@ -58,7 +58,7 @@ def createRedditsSubdirectories(pagesMainDir): def createSubdirectories(pagesDir): - currentDateDir = pagesDir + '/' + str("%02d" %date.today().month) + str("%02d" %date.today().day) + str("%04d" %date.today().year) + currentDateDir = pagesDir + '/' + CURRENT_DATE if not os.path.isdir(currentDateDir): os.mkdir(currentDateDir) @@ -79,19 +79,19 @@ def createSubdirectories(pagesDir): os.mkdir(descReadDir) -#main method +# main method if __name__ == '__main__': - #assignment from forumsList.txt + # assignment from forumsList.txt forumsList = getForums() - #get forum from forumsList + # get forum from forumsList for forum in forumsList: forum = forum.replace('\n','') print("Creating listing and description directories ...") createDirectory(forum) - time.sleep(5) #wait for directories to be created + time.sleep(5) # wait for directories to be created input("Directories created successfully. Press ENTER to continue\n") if forum == "BestCardingWorld": @@ -103,7 +103,6 @@ if __name__ == '__main__': elif forum == "CrackingPro": crawlerCrackingPro() - print("Scraping process completed successfully!") diff --git a/Forums/Initialization/geckodriver.log b/Forums/Initialization/geckodriver.log index c206435..15928b8 100644 --- a/Forums/Initialization/geckodriver.log +++ b/Forums/Initialization/geckodriver.log @@ -3963,3 +3963,122 @@ JavaScript error: resource://gre/modules/ExtensionTelemetry.jsm, line 109: Error JavaScript error: resource://gre/modules/ExtensionTelemetry.jsm, line 113: Error: TelemetryStopwatch: finishing nonexisting stopwatch. Histogram: "WEBEXT_CONTENT_SCRIPT_INJECTION_MS_BY_ADDONID", key: "{73a6fe31-595d-460b-a920-fcc0f8843232}" JavaScript error: resource://gre/modules/ExtensionTelemetry.jsm, line 109: Error: TelemetryStopwatch: finishing nonexisting stopwatch. Histogram: "WEBEXT_CONTENT_SCRIPT_INJECTION_MS", key: "" JavaScript error: resource://gre/modules/ExtensionTelemetry.jsm, line 113: Error: TelemetryStopwatch: finishing nonexisting stopwatch. Histogram: "WEBEXT_CONTENT_SCRIPT_INJECTION_MS_BY_ADDONID", key: "{73a6fe31-595d-460b-a920-fcc0f8843232}" +1687240079948 geckodriver INFO Listening on 127.0.0.1:50448 +1687240084735 mozrunner::runner INFO Running command: "C:\\Users\\calsyslab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50449" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileuYe2AP" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: init()" +console.log: "TorConnect: Entering Initial state" +console.log: "TorConnect: Observed profile-after-change" +console.log: "TorConnect: Observing topic 'TorProcessExited'" +console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: Observing topic 'torsettings:ready'" +console.log: "TorSettings: Observed profile-after-change" +1687240085868 Marionette INFO Marionette enabled +console.log: "TorConnect: Will load after bootstrap => [about:blank]" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:50449/devtools/browser/e85e6865-1f97-480a-8e46-778271184a87 +1687240090364 Marionette INFO Listening on port 50454 +1687240090846 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/, line 2: ReferenceError: $ is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 2: ReferenceError: $ is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 5: ReferenceError: lang is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 9: ReferenceError: use_xmlhttprequest is not defined +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\calsyslab\AppData\Local\Temp\rust_mozprofileuYe2AP\thumbnails) because it does not exist +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/forumdisplay.php?fid=86, line 3: ReferenceError: lang is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/jscripts/jeditable/jeditable.min.js, line 38: ReferenceError: jQuery is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/jscripts/inline_edit.js?ver=1808, line 6: ReferenceError: $ is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/showthread.php?tid=2628, line 6: ReferenceError: lang is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/jscripts/report.js?ver=1804, line 4: ReferenceError: $ is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/jscripts/jeditable/jeditable.min.js, line 38: ReferenceError: jQuery is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/jscripts/thread.js?ver=1809, line 4: ReferenceError: $ is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/showthread.php?tid=2628, line 19: ReferenceError: use_xmlhttprequest is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/showthread.php?tid=2628, line 25: ReferenceError: $ is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/showthread.php?tid=2628&page=2, line 6: ReferenceError: lang is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/jscripts/report.js?ver=1804, line 4: ReferenceError: $ is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/jscripts/jeditable/jeditable.min.js, line 38: ReferenceError: jQuery is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/jscripts/thread.js?ver=1809, line 4: ReferenceError: $ is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/showthread.php?tid=2628&page=2, line 19: ReferenceError: use_xmlhttprequest is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/showthread.php?tid=2628&page=2, line 25: ReferenceError: $ is not defined +1687240218310 Marionette INFO Stopped listening on port 50454 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first. +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\calsyslab\AppData\Local\Temp\rust_mozprofileuYe2AP\thumbnails) because it does not exist +1687240220095 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1687240311209 geckodriver INFO Listening on 127.0.0.1:50519 +1687240315070 mozrunner::runner INFO Running command: "C:\\Users\\calsyslab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "50520" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofiletzrkDs" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: init()" +console.log: "TorConnect: Entering Initial state" +console.log: "TorConnect: Observed profile-after-change" +console.log: "TorConnect: Observing topic 'TorProcessExited'" +console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: Observing topic 'torsettings:ready'" +console.log: "TorSettings: Observed profile-after-change" +1687240315958 Marionette INFO Marionette enabled +console.log: "TorConnect: Will load after bootstrap => [about:blank]" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:50520/devtools/browser/4b6276ea-c420-4b6d-b4bc-fda679f97800 +1687240317156 Marionette INFO Listening on port 50525 +1687240317256 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/, line 2: ReferenceError: $ is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 2: ReferenceError: $ is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 5: ReferenceError: lang is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/member.php?action=login, line 9: ReferenceError: use_xmlhttprequest is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/forumdisplay.php?fid=86, line 3: ReferenceError: lang is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/jscripts/jeditable/jeditable.min.js, line 38: ReferenceError: jQuery is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/jscripts/inline_edit.js?ver=1808, line 6: ReferenceError: $ is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/showthread.php?tid=2628, line 6: ReferenceError: lang is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/jscripts/report.js?ver=1804, line 4: ReferenceError: $ is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/jscripts/jeditable/jeditable.min.js, line 38: ReferenceError: jQuery is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/jscripts/thread.js?ver=1809, line 4: ReferenceError: $ is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/showthread.php?tid=2628, line 19: ReferenceError: use_xmlhttprequest is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/showthread.php?tid=2628, line 25: ReferenceError: $ is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/forumdisplay.php?fid=86, line 3: ReferenceError: lang is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/jscripts/jeditable/jeditable.min.js, line 38: ReferenceError: jQuery is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/jscripts/inline_edit.js?ver=1808, line 6: ReferenceError: $ is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/forumdisplay.php?fid=86&page=2, line 3: ReferenceError: lang is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/jscripts/jeditable/jeditable.min.js, line 38: ReferenceError: jQuery is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/jscripts/inline_edit.js?ver=1808, line 6: ReferenceError: $ is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/showthread.php?tid=16404, line 6: ReferenceError: lang is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/jscripts/report.js?ver=1804, line 4: ReferenceError: $ is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/jscripts/jeditable/jeditable.min.js, line 38: ReferenceError: jQuery is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/jscripts/thread.js?ver=1809, line 4: ReferenceError: $ is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/showthread.php?tid=16404, line 19: ReferenceError: use_xmlhttprequest is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/showthread.php?tid=16404, line 25: ReferenceError: $ is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/forumdisplay.php?fid=86&page=2, line 3: ReferenceError: lang is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/jscripts/jeditable/jeditable.min.js, line 38: ReferenceError: jQuery is not defined +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/jscripts/inline_edit.js?ver=1808, line 6: ReferenceError: $ is not defined +1687240409940 Marionette INFO Stopped listening on port 50525 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] + +###!!! [Parent][MessageChannel] Error: (msgtype=0x140007,name=PBackgroundLSDatabase::Msg_RequestAllowToClose) Channel error: cannot send/recv + +[Parent 1036, IPC I/O Parent] WARNING: file /var/tmp/build/firefox-b6010b1466c9/ipc/chromium/src/base/process_util_win.cc:167 +!!! error running onStopped callback: TypeError: callback is not a function +JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first. +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\calsyslab\AppData\Local\Temp\rust_mozprofiletzrkDs\thumbnails) because it does not exist + +###!!! [Child][MessageChannel] Error: (msgtype=0x5D0005,name=PImageBridge::Msg_WillClose) Channel error: cannot send/recv + +1687240410572 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 diff --git a/Forums/Initialization/prepare_parser.py b/Forums/Initialization/prepare_parser.py index 23d97f1..2efb84d 100644 --- a/Forums/Initialization/prepare_parser.py +++ b/Forums/Initialization/prepare_parser.py @@ -9,7 +9,7 @@ from Forums.BestCardingWorld.parser import * from Forums.CryptBB.parser import * from Forums.Classifier.classify_product import predict -#from DarkWebMining_Sample.Forums.Classifier.classify_product import predict_semi +# from DarkWebMining_Sample.Forums.Classifier.classify_product import predict_semi # determines if forum is russian, not really used now but maybe later @@ -62,9 +62,9 @@ def getPosts(posts): #uses db connection , another program, methods to persists values to the correct categories #@param: row is the list of entries for this instance, cur is the db connection object -def persist_data(row, cur): +def persist_data(url, row, cur): - forum = create_forum(cur, row) + forum = create_forum(cur, row, url) board = create_board(cur, row, forum) @@ -77,15 +77,13 @@ def persist_data(row, cur): #main method for this program, what actually gets the parsed info from the parser, and persists them into the db #calls the different parser methods here depending on the type of html page -def new_parse(forum, createLog): +def new_parse(forum, url, createLog): - print("Parsing The " + forum + " Forum and conduct data classification to store the information in the database.") - - crawlerDate = date.today() + from Forums.Initialization.forums_mining import CURRENT_DATE - ini = time.time() + print("Parsing The " + forum + " Forum and conduct data classification to store the information in the database.") - global site + # ini = time.time() # Connecting to the database con = connectDataBase() @@ -96,27 +94,26 @@ def new_parse(forum, createLog): nError = 0 - lines = [] #lines.clear() - lns = [] #lns.clear() + lines = [] # listing pages + lns = [] # description pages detPage = {} - rw = [] # Creating the log file for each Forum if createLog: - if not os.path.exists("./" + forum + "/Logs/" + forum + "_" + str("%02d" %crawlerDate.month) + str("%02d" %crawlerDate.day) + str("%04d" %crawlerDate.year) + ".log"): - logFile = open("./" + forum + "/Logs/" + forum + "_" + str("%02d" %crawlerDate.today().month) + str("%02d" %crawlerDate.day) + str("%04d" %crawlerDate.year) + ".log", "w") + if not os.path.exists("./" + forum + "/Logs/" + forum + "_" + CURRENT_DATE + ".log"): + logFile = open("./" + forum + "/Logs/" + forum + "_" + CURRENT_DATE + ".log", "w") else: - print("Files of the date " + str("%02d" %crawlerDate.today().month) + str("%02d" %crawlerDate.today().day) + str("%04d" %crawlerDate.today().year) + - " from the Forum " + forum + " were already read. Delete the referent information in the Data Base and also delete the log file " - "in the _Logs folder to read files from this Forum of this date again.") + print("Files of the date " + CURRENT_DATE + " from the Forum " + forum + + " were already read. Delete the referent information in the Data Base and also delete the log file" + " in the _Logs folder to read files from this Forum of this date again.") raise SystemExit # Reading the Listing Html Pages - for fileListing in glob.glob(os.path.join (os.getcwd().replace("Initialization","") + forum + "\\HTML_Pages\\" + str("%02d" %crawlerDate.month) + str("%02d" %crawlerDate.day) + str("%04d" %crawlerDate.year) + "\\Listing" ,'*.html')): + for fileListing in glob.glob(os.path.join("..\\" + forum + "\\HTML_Pages\\" + CURRENT_DATE + "\\Listing", '*.html')): lines.append(fileListing) # Reading the Description Html Pages - for fileDescription in glob.glob(os.path.join (os.getcwd().replace("Initialization","") + forum + "\\HTML_Pages\\" + str("%02d" %crawlerDate.month) + str("%02d" %crawlerDate.day) + str("%04d" %crawlerDate.year) + "\\Description" ,'*.html')): + for fileDescription in glob.glob(os.path.join("..\\" + forum + "\\HTML_Pages\\" + CURRENT_DATE + "\\Description" ,'*.html')): lns.append(fileDescription) # Parsing the Description Pages and put the tag's content into a dictionary (Hash table) @@ -218,9 +215,7 @@ def new_parse(forum, createLog): rec = rec.split(',') # key = u"Top:" + rec[1].upper().strip() + u" User:" + rec[5].upper().strip() - # key = rec[16] - url = ''.join(e for e in rec[6] if e.isalnum()) - key = u"Url:" + url + key = u"Url:" + cleanLink(rec[6]) if key in detPage: @@ -237,7 +232,7 @@ def new_parse(forum, createLog): # Persisting the information in the database try: - persist_data(tuple(rec), cur) + persist_data(url, tuple(rec), cur) con.commit() except: diff --git a/Forums/Utilities/utilities.py b/Forums/Utilities/utilities.py index 9d64cb6..d8ca9eb 100644 --- a/Forums/Utilities/utilities.py +++ b/Forums/Utilities/utilities.py @@ -160,7 +160,7 @@ def cleanLink(originalLink): return originalLink -def organizeTopics(forum, nm, topic, board, author, views, posts, href, addDate): +def organizeTopics(forum, nm, board, author, topic, views, posts, href, addDate): day = time.strftime("%m/%d/%Y") ahora = time.strftime("%I:%M:%S") diff --git a/MarketPlaces/DB_Connection/db_connection.py b/MarketPlaces/DB_Connection/db_connection.py index 9cabf34..97296e3 100644 --- a/MarketPlaces/DB_Connection/db_connection.py +++ b/MarketPlaces/DB_Connection/db_connection.py @@ -2,15 +2,21 @@ __author__ = 'DarkWeb' import psycopg2 import traceback -import time -from datetime import date +import configparser def connectDataBase(): try: - return psycopg2.connect(host='localhost', user='postgres', password='password', dbname='darkweb_markets_forums') + config = configparser.ConfigParser() + config.read('../../setup.ini') + ip = config.get('PostgreSQL', 'ip') + username = config.get('PostgreSQL', 'username') + password = config.get('PostgreSQL', 'password') + database = config.get('PostgreSQL', 'database') + + return psycopg2.connect(host=ip, user=username, password=password, dbname=database) except: @@ -95,7 +101,7 @@ def getLastVendor(cur): print (trace) -def create_marketPlace(cur, row): +def create_marketPlace(cur, row, url): marketId = verifyMarketPlace(cur, row[0]) @@ -105,7 +111,7 @@ def create_marketPlace(cur, row): sql = "Insert into marketplaces (market_id, name_market, url_market, dateinserted_market) " \ "Values (%s, %s, %s, %s)" - recset = [marketId, row[0], None, row[21]] + recset = [marketId, row[0], url, row[21]] cur.execute(sql, recset) diff --git a/MarketPlaces/Initialization/geckodriver.log b/MarketPlaces/Initialization/geckodriver.log index 51d45ff..7f95777 100644 --- a/MarketPlaces/Initialization/geckodriver.log +++ b/MarketPlaces/Initialization/geckodriver.log @@ -6073,3 +6073,35 @@ unwatchForTargets()@TargetList.jsm:37 destructor()@TargetList.jsm:109 stop()@CDP.jsm:104 close()@RemoteAgent.jsm:138 +1687245533907 geckodriver INFO Listening on 127.0.0.1:62051 +1687245536832 mozrunner::runner INFO Running command: "C:\\Users\\calsyslab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "62052" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofileuMGaeY" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: init()" +console.log: "TorConnect: Entering Initial state" +console.log: "TorConnect: Observed profile-after-change" +console.log: "TorConnect: Observing topic 'TorProcessExited'" +console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: Observing topic 'torsettings:ready'" +console.log: "TorSettings: Observed profile-after-change" +1687245537956 Marionette INFO Marionette enabled +console.log: "TorConnect: Will load after bootstrap => [about:blank]" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:62052/devtools/browser/9cf17e56-2fb1-468d-b65e-15c4de4eaa64 +1687245540759 Marionette INFO Listening on port 49935 +1687245540897 RemoteAgent WARN TLS certificate errors will be ignored for this session +1687245639406 Marionette INFO Stopped listening on port 49935 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first. +JavaScript error: resource://gre/modules/PageThumbs.jsm, line 709: AbortError: IOUtils.profileBeforeChange getter: IOUtils: profileBeforeChange phase has already finished +1687245650576 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 diff --git a/MarketPlaces/Initialization/marketsList.txt b/MarketPlaces/Initialization/marketsList.txt index 87f811c..b85ae71 100644 --- a/MarketPlaces/Initialization/marketsList.txt +++ b/MarketPlaces/Initialization/marketsList.txt @@ -1 +1 @@ -ThiefWorld \ No newline at end of file +Tor2door \ No newline at end of file diff --git a/MarketPlaces/Initialization/markets_mining.py b/MarketPlaces/Initialization/markets_mining.py index 42bb51c..3073612 100644 --- a/MarketPlaces/Initialization/markets_mining.py +++ b/MarketPlaces/Initialization/markets_mining.py @@ -12,6 +12,8 @@ from MarketPlaces.ThiefWorld.crawler_selenium import crawler as crawlerThiefWorl import time +CURRENT_DATE = str("%02d" % date.today().month) + str("%02d" % date.today().day) + str("%04d" % date.today().year) + # reads list of marketplaces def getMarkets(): @@ -26,12 +28,10 @@ def createDirectory(mkt): # Package should already be there, holding crawler and parser pagesDir = '../' + mkt + '/HTML_Pages' - # sharedFolderPath = r'\\VBoxSvr\VM_Files_(shared)' - # pagesDir = os.path.join(sharedFolderPath, 'HTML/MarketPlaces/' + mkt + '/HTML_Pages') if not os.path.isdir(pagesDir): os.makedirs(pagesDir) - currentDateDir = pagesDir + '/' + str("%02d" %date.today().month) + str("%02d" %date.today().day) + str("%04d" %date.today().year) + currentDateDir = pagesDir + '/' + CURRENT_DATE if not os.path.isdir(currentDateDir): os.mkdir(currentDateDir) diff --git a/MarketPlaces/Initialization/prepare_parser.py b/MarketPlaces/Initialization/prepare_parser.py index 2389834..de13899 100644 --- a/MarketPlaces/Initialization/prepare_parser.py +++ b/MarketPlaces/Initialization/prepare_parser.py @@ -60,55 +60,52 @@ def mergePages(rmm, rec): return rec -def persist_data(row, cur): +def persist_data(url, row, cur): - marketPlace = create_marketPlace(cur, row) + marketPlace = create_marketPlace(cur, row, url) vendor = create_vendor(cur, row, marketPlace) create_items(cur, row, marketPlace, vendor) -def new_parse(marketPlace, createLog): +def new_parse(marketPlace, url, createLog): - print("Parsing the " + marketPlace + " marketplace and conduct data classification to store the information in the database.") + from MarketPlaces.Initialization.markets_mining import CURRENT_DATE - crawlerDate = date.today() + print("Parsing the " + marketPlace + " marketplace and conduct data classification to store the information in the database.") # ini = time.time() - global site - - #Connecting to the database + # Connecting to the database con = connectDataBase() cur = con.cursor() - #Creating the tables (The database should be created manually) + # Creating the tables (The database should be created manually) create_database(cur, con) nError = 0 - lines = [] #lines.clear() - lns = [] #lns.clear() + lines = [] # listing pages + lns = [] # description pages detPage = {} - rw = [] #Creating the log file for each Market Place if createLog: - if not os.path.exists("./" + marketPlace + "/Logs/" + marketPlace + "_" + str("%02d" %crawlerDate.month) + str("%02d" %crawlerDate.day) + str("%04d" %crawlerDate.year) + ".log"): - logFile = open("./" + marketPlace + "/Logs/" + marketPlace + "_" + str("%02d" %crawlerDate.month) + str("%02d" %crawlerDate.day) + str("%04d" %crawlerDate.year) + ".log", "w") + if not os.path.exists("./" + marketPlace + "/Logs/" + marketPlace + "_" + CURRENT_DATE + ".log"): + logFile = open("./" + marketPlace + "/Logs/" + marketPlace + "_" + CURRENT_DATE + ".log", "w") else: - print("Files of the date " + str("%02d" %crawlerDate.month) + "/" + str("%02d" %crawlerDate.day) + "/" + str("%04d" %crawlerDate.year) + - " from the Market Place " + marketPlace + " were already read. Delete the referent information in the Data Base and also delete the log file " - "in the _Logs folder to read files from this Market Place of this date again.") + print("Files of the date " + CURRENT_DATE + " from the Market Place " + marketPlace + + " were already read. Delete the referent information in the Data Base and also delete the log file" + " in the _Logs folder to read files from this Market Place of this date again.") raise SystemExit # Reading the Listing Html Pages - for fileListing in glob.glob(os.path.join (os.getcwd().replace("Initialization","") + marketPlace + "\\HTML_Pages\\" + str("%02d" %crawlerDate.month) + str("%02d" %crawlerDate.day) + str("%04d" %crawlerDate.year) + "\\Listing" ,'*.html')): + for fileListing in glob.glob(os.path.join("..\\" + marketPlace + "\\HTML_Pages\\" + CURRENT_DATE + "\\Listing", '*.html')): lines.append(fileListing) # Reading the Description Html Pages - for fileDescription in glob.glob(os.path.join (os.getcwd().replace("Initialization","") + marketPlace + "\\HTML_Pages\\" + str("%02d" %crawlerDate.month) + str("%02d" %crawlerDate.day) + str("%04d" %crawlerDate.year) + "\\Description" ,'*.html')): + for fileDescription in glob.glob(os.path.join("..\\" + marketPlace + "\\HTML_Pages\\" + CURRENT_DATE + "\\Description", '*.html')): lns.append(fileDescription) # Parsing the Description Pages and put the tag's content into a dictionary (Hash table) @@ -214,9 +211,7 @@ def new_parse(marketPlace, createLog): # key = rec[23] # key = u"Pr:" + rec[1].upper()[:list_lim1] + u" Vendor:" + rec[18].upper()[:list_lim2] - # key = u"Pr:" + rec[1].upper() - url = ''.join(e for e in rec[20] if e.isalnum()) - key = u"Url:" + url + key = u"Url:" + cleanLink(rec[20]) # if the associated description page is parsed if key in detPage: @@ -233,7 +228,7 @@ def new_parse(marketPlace, createLog): # Persisting the information in the database try: - persist_data(tuple(rec), cur) + persist_data(url, tuple(rec), cur) con.commit() except: diff --git a/MarketPlaces/ThiefWorld/crawler_selenium.py b/MarketPlaces/ThiefWorld/crawler_selenium.py index 34d606d..3d3c28a 100644 --- a/MarketPlaces/ThiefWorld/crawler_selenium.py +++ b/MarketPlaces/ThiefWorld/crawler_selenium.py @@ -15,14 +15,17 @@ from selenium.webdriver.common.by import By from PIL import Image import urllib.parse as urlparse -import os, time +import os, re, time from datetime import date import subprocess +import configparser from bs4 import BeautifulSoup from MarketPlaces.Initialization.prepare_parser import new_parse from MarketPlaces.ThiefWorld.parser import thiefworld_links_parser from MarketPlaces.Utilities.utilities import cleanHTML +config = configparser.ConfigParser() +config.read('../../setup.ini') counter = 1 baseURL = 'http://qsw7iurcrdwyml5kg4oxbmtqrcnpxiag3iumdarefzeunnyc2dnyljad.onion/' @@ -31,7 +34,7 @@ baseURL = 'http://qsw7iurcrdwyml5kg4oxbmtqrcnpxiag3iumdarefzeunnyc2dnyljad.onion #acts like the main method for the crawler, another function at the end of this code calls this function later def startCrawling(): opentor() - mktName = getMKTName() + # mktName = getMKTName() driver = getAccess() if driver != 'down': @@ -42,7 +45,7 @@ def startCrawling(): print(driver.current_url, e) closetor(driver) - #new_parse(mktName, False) + # new_parse(mktName, False) # Opens Tor Browser @@ -50,8 +53,7 @@ def startCrawling(): def opentor(): global pid print("Connecting Tor...") - path = open('../../path.txt').readline().strip() - pro = subprocess.Popen(path) + pro = subprocess.Popen(config.get('TOR', 'firefox_binary_path')) pid = pro.pid time.sleep(7.5) input('Tor Connected. Press ENTER to continue\n') @@ -61,7 +63,7 @@ def opentor(): # Returns the name of the website #return: name of site in string type def getMKTName(): - name = 'TheifWorld' + name = 'ThiefWorld' return name @@ -87,12 +89,9 @@ def closetor(driver): # Creates FireFox 'driver' and configure its 'Profile' # to use Tor proxy and socket def createFFDriver(): - file = open('../../path.txt', 'r') - lines = file.readlines() + ff_binary = FirefoxBinary(config.get('TOR', 'firefox_binary_path')) - ff_binary = FirefoxBinary(lines[0].strip()) - - ff_prof = FirefoxProfile(lines[1].strip()) + ff_prof = FirefoxProfile(config.get('TOR', 'firefox_profile_path')) ff_prof.set_preference("places.history.enabled", False) ff_prof.set_preference("privacy.clearOnShutdown.offlineApps", True) ff_prof.set_preference("privacy.clearOnShutdown.passwords", True) @@ -114,7 +113,7 @@ def createFFDriver(): ff_prof.set_preference("javascript.enabled", False) ff_prof.update_preferences() - service = Service(lines[2].strip()) + service = Service(config.get('TOR', 'geckodriver_path')) driver = webdriver.Firefox(firefox_binary=ff_binary, firefox_profile=ff_prof, service=service) @@ -162,15 +161,12 @@ def savePage(page, url): # Gets the full path of the page to be saved along with its appropriate file name #@param: raw url as crawler crawls through every site def getFullPathName(url): + from MarketPlaces.Initialization.markets_mining import CURRENT_DATE fileName = getNameFromURL(url) if isDescriptionLink(url): - fullPath = r'..\ThiefWorld\HTML_Pages\\' + str( - "%02d" % date.today().month) + str("%02d" % date.today().day) + str( - "%04d" % date.today().year) + r'\\' + r'Description\\' + fileName + '.html' + fullPath = r'..\ThiefWorld\HTML_Pages\\' + CURRENT_DATE + r'\\Description\\' + fileName + '.html' else: - fullPath = r'..\ThiefWorld\HTML_Pages\\' + str( - "%02d" % date.today().month) + str("%02d" % date.today().day) + str( - "%04d" % date.today().year) + r'\\' + r'Listing\\' + fileName + '.html' + fullPath = r'..\ThiefWorld\HTML_Pages\\' + CURRENT_DATE + r'\\Listing\\' + fileName + '.html' return fullPath @@ -191,14 +187,15 @@ def getNameFromURL(url): #as you can see they are categories of products def getInterestedLinks(): links = [] + # Hacking and DDOS links.append('http://qsw7iurcrdwyml5kg4oxbmtqrcnpxiag3iumdarefzeunnyc2dnyljad.onion/catalog/35') # # Carding Manuals - links.append('http://qsw7iurcrdwyml5kg4oxbmtqrcnpxiag3iumdarefzeunnyc2dnyljad.onion/catalog/20') + # links.append('http://qsw7iurcrdwyml5kg4oxbmtqrcnpxiag3iumdarefzeunnyc2dnyljad.onion/catalog/20') # # Software - links.append('http://qsw7iurcrdwyml5kg4oxbmtqrcnpxiag3iumdarefzeunnyc2dnyljad.onion/catalog/37') - # #Database - links.append('http://qsw7iurcrdwyml5kg4oxbmtqrcnpxiag3iumdarefzeunnyc2dnyljad.onion/catalog/38') + # links.append('http://qsw7iurcrdwyml5kg4oxbmtqrcnpxiag3iumdarefzeunnyc2dnyljad.onion/catalog/37') + # # Database + # links.append('http://qsw7iurcrdwyml5kg4oxbmtqrcnpxiag3iumdarefzeunnyc2dnyljad.onion/catalog/38') return links @@ -238,13 +235,13 @@ def crawlForum(driver): savePage(driver.page_source, item) driver.back() - # # comment out - # break - # - # # # comment out - # if count == 1: - # count = 0 - # break + # comment out + break + + # comment out + if count == 1: + count = 0 + break try: link = driver.find_element(by=By.XPATH, value= diff --git a/MarketPlaces/Tor2door/crawler_selenium.py b/MarketPlaces/Tor2door/crawler_selenium.py index b7e7937..baef719 100644 --- a/MarketPlaces/Tor2door/crawler_selenium.py +++ b/MarketPlaces/Tor2door/crawler_selenium.py @@ -15,41 +15,42 @@ from selenium.webdriver.support.ui import WebDriverWait from PIL import Image import urllib.parse as urlparse -import os, time -from datetime import date +import os, re, time import subprocess +import configparser from bs4 import BeautifulSoup from MarketPlaces.Initialization.prepare_parser import new_parse from MarketPlaces.Tor2door.parser import tor2door_links_parser from MarketPlaces.Utilities.utilities import cleanHTML +config = configparser.ConfigParser() +config.read('../../setup.ini') counter = 1 -baseURL = 'http://http://yzrrne3pveltulbavydr2kiashvlnysdwclwmklo6cyjuqpxi7ku4xqd.onion' +baseURL = 'http://yzrrne3pveltulbavydr2kiashvlnysdwclwmklo6cyjuqpxi7ku4xqd.onion' # Opens Tor Browser, crawls the website def startCrawling(): - opentor() - # marketName = getMarketName() - driver = getAccess() - - if driver != 'down': - try: - login(driver) - crawlForum(driver) - except Exception as e: - print(driver.current_url, e) - closetor(driver) - - # new_parse(marketName, False) + # opentor() + marketName = getMarketName() + # driver = getAccess() + # + # if driver != 'down': + # try: + # login(driver) + # crawlForum(driver) + # except Exception as e: + # print(driver.current_url, e) + # closetor(driver) + # + new_parse(marketName, baseURL, False) # Opens Tor Browser def opentor(): global pid print("Connecting Tor...") - path = open('../../path.txt').readline().strip() - pro = subprocess.Popen(path) + pro = subprocess.Popen(config.get('TOR', 'firefox_binary_path')) pid = pro.pid time.sleep(7.5) input('Tor Connected. Press ENTER to continue\n') @@ -130,12 +131,9 @@ def closetor(driver): # Creates FireFox 'driver' and configure its 'Profile' # to use Tor proxy and socket def createFFDriver(): - file = open('../../path.txt', 'r') - lines = file.readlines() - - ff_binary = FirefoxBinary(lines[0].strip()) + ff_binary = FirefoxBinary(config.get('TOR', 'firefox_binary_path')) - ff_prof = FirefoxProfile(lines[1].strip()) + ff_prof = FirefoxProfile(config.get('TOR', 'firefox_profile_path')) ff_prof.set_preference("places.history.enabled", False) ff_prof.set_preference("privacy.clearOnShutdown.offlineApps", True) ff_prof.set_preference("privacy.clearOnShutdown.passwords", True) @@ -157,7 +155,7 @@ def createFFDriver(): ff_prof.set_preference("javascript.enabled", False) ff_prof.update_preferences() - service = Service(executable_path=lines[2].strip()) + service = Service(config.get('TOR', 'geckodriver_path')) driver = webdriver.Firefox(firefox_binary=ff_binary, firefox_profile=ff_prof, service=service) @@ -186,15 +184,12 @@ def savePage(page, url): # Gets the full path of the page to be saved along with its appropriate file name def getFullPathName(url): + from MarketPlaces.Initialization.markets_mining import CURRENT_DATE fileName = getNameFromURL(url) if isDescriptionLink(url): - fullPath = r'..\Tor2door\HTML_Pages\\' + str( - "%02d" % date.today().month) + str("%02d" % date.today().day) + str( - "%04d" % date.today().year) + r'\\' + r'Description\\' + fileName + '.html' + fullPath = r'..\Tor2door\HTML_Pages\\' + CURRENT_DATE + r'\\Description\\' + fileName + '.html' else: - fullPath = r'..\Tor2door\HTML_Pages\\' + str( - "%02d" % date.today().month) + str("%02d" % date.today().day) + str( - "%04d" % date.today().year) + r'\\' + r'Listing\\' + fileName + '.html' + fullPath = r'..\Tor2door\HTML_Pages\\' + CURRENT_DATE + r'\\Listing\\' + fileName + '.html' return fullPath diff --git a/MarketPlaces/Tor2door/parser.py b/MarketPlaces/Tor2door/parser.py index 105fc99..f4a4c07 100644 --- a/MarketPlaces/Tor2door/parser.py +++ b/MarketPlaces/Tor2door/parser.py @@ -12,10 +12,10 @@ def tor2door_description_parser(soup): # Fields to be parsed - vendor = "-1" # 0 Vendor_Name + vendor = "-1" # 0 *Vendor_Name success = "-1" # 1 Vendor_Successful_Transactions rating_vendor = "-1" # 2 Vendor_Rating - name = "-1" # 3 Product_Name + name = "-1" # 3 *Product_Name describe = "-1" # 4 Product_Description CVE = "-1" # 5 Product_CVE_Classification (Common Vulnerabilities and Exposures) MS = "-1" # 6 Product_MS_Classification (Microsoft Security) @@ -118,28 +118,28 @@ def tor2door_description_parser(soup): def tor2door_listing_parser(soup): # Fields to be parsed - nm = 0 # Total_Products (Should be Integer) - mktName = "Tor2door" # 0 Marketplace_Name - vendor = [] # 18 Vendor y - rating_vendor = [] # 19 Vendor_Rating - success = [] # 20 Vendor_Successful_Transactions - name = [] # 1 Product_Name y - CVE = [] # 2 Product_CVE_Classification (Common Vulnerabilities and Exposures) - MS = [] # 3 Product_MS_Classification (Microsoft Security) - category = [] # 4 Product_Category y - describe = [] # 5 Product_Description - views = [] # 7 Product_Number_Of_Views - reviews = [] # 7 Product_Number_Of_Reviews - rating_item = [] # 8 Product_Rating - addDate = [] # 9 Product_AddDate - BTC = [] # 11 Product_BTC_SellingPrice - USD = [] # 12 Product_USD_SellingPrice y - EURO = [] # 13 Product_EURO_SellingPrice - sold = [] # 14 Product_QuantitySold - qLeft =[] # 15 Product_QuantityLeft - shipFrom = [] # 16 Product_ShippedFrom - shipTo = [] # 17 Product_ShippedTo - href = [] # 24 Product_Links + nm = 0 # *Total_Products (Should be Integer) + mktName = "Tor2door" # 0 *Marketplace_Name + vendor = [] # 1 *Vendor y + rating_vendor = [] # 2 Vendor_Rating + success = [] # 3 Vendor_Successful_Transactions + name = [] # 4 *Product_Name y + CVE = [] # 5 Product_CVE_Classification (Common Vulnerabilities and Exposures) + MS = [] # 6 Product_MS_Classification (Microsoft Security) + category = [] # 7 Product_Category y + describe = [] # 8 Product_Description + views = [] # 9 Product_Number_Of_Views + reviews = [] # 10 Product_Number_Of_Reviews + rating_item = [] # 11 Product_Rating + addDate = [] # 12 Product_AddDate + BTC = [] # 13 Product_BTC_SellingPrice + USD = [] # 14 Product_USD_SellingPrice y + EURO = [] # 15 Product_EURO_SellingPrice + sold = [] # 16 Product_QuantitySold + qLeft =[] # 17 Product_QuantityLeft + shipFrom = [] # 18 Product_ShippedFrom + shipTo = [] # 19 Product_ShippedTo + href = [] # 20 Product_Links listing = soup.findAll('div', {"class": "card product-card mb-3"}) @@ -160,7 +160,6 @@ def tor2door_listing_parser(soup): # Adding the url to the list of urls link = bae[0].get('href') - link = cleanLink(link) href.append(link) # Finding Product Name diff --git a/path.txt b/path.txt deleted file mode 100644 index 3992963..0000000 --- a/path.txt +++ /dev/null @@ -1,3 +0,0 @@ -C:\Users\Helium\Desktop\Tor Browser\Browser\firefox.exe -C:\Users\Helium\Desktop\Tor Browser\Browser\TorBrowser\Data\Browser\profile.default -C:\Users\Helium\PycharmProjects\dw_pipeline_test\selenium\geckodriver.exe \ No newline at end of file diff --git a/setup.ini b/setup.ini new file mode 100644 index 0000000..38c2347 --- /dev/null +++ b/setup.ini @@ -0,0 +1,14 @@ +[TOR] +firefox_binary_path = C:\Users\calsyslab\Desktop\Tor Browser\Browser\firefox.exe +firefox_profile_path = C:\Users\calsyslab\Desktop\Tor Browser\Browser\TorBrowser\Data\Browser\profile.default +geckodriver_path = C:\Users\calsyslab\Projects\dw_pipeline_test\selenium\geckodriver.exe + +[Project] +project_directory = C:\Users\calsyslab\Projects\dw_pipeline_test +shared_folder = \\VBoxSvr\VM_Files_(shared) + +[PostgreSQL] +ip = localhost +username = postgres +password = password +database = darkweb_markets_forums \ No newline at end of file