diff --git a/.idea/.gitignore b/.idea/.gitignore index 7b08725..1b8473e 100644 --- a/.idea/.gitignore +++ b/.idea/.gitignore @@ -2,6 +2,7 @@ /shelf/ /workspace.xml /selenium/geckodriver.exe +setup.ini *.html *.log *.png diff --git a/.idea/.name b/.idea/.name new file mode 100644 index 0000000..f6606ac --- /dev/null +++ b/.idea/.name @@ -0,0 +1 @@ +forums_mining.py \ No newline at end of file diff --git a/Forums/AbyssForum/HTML_Pages/06262023/Description/viewtopicphpt1402.html b/Forums/AbyssForum/HTML_Pages/06262023/Description/viewtopicphpt1402.html new file mode 100644 index 0000000..19fdcdc --- /dev/null +++ b/Forums/AbyssForum/HTML_Pages/06262023/Description/viewtopicphpt1402.html @@ -0,0 +1,586 @@ + + + + + + + + + + + +Get List For WinPot malware,cutlet maker, Yoda and UPX type ATM jackpotting. - Abyss Forum| forum, drugs,guns,porn,hacking,crypto + + + + + + + + + + + + + + + + + + +{"@context":"https://schema.org","@type":"DiscussionForumPosting","url":"http://qyvjopwdgjq52ehsx6paonv2ophy3p4ivfkul4svcaw6qxlzsaboyjid.onion/viewtopic.php?t=1402","headline":"Get List For WinPot malware,cutlet maker, Yoda and UPX type ATM jackpotting.","description":"Get List For WinPot malware,cutlet maker, Yoda and UPX type ATM jackpotting WinPot Malware WinPot malware ATMii Malware Cutler Malware Buy ATM jackpotting virus","author":{"@type":"Person","name":"darkclonee"},"datePublished":"2023-04-29T22:33:53+00:00","articleSection":"Hacked Database","publisher":{"@type":"Organization","name":"Abyss Forum| forum, drugs,guns,porn,hacking,crypto","url":"http://qyvjopwdgjq52ehsx6paonv2ophy3p4ivfkul4svcaw6qxlzsaboyjid.onion"}} + + + + + + + + + + + +
+ + + + + +
+ + +

Get List For WinPot malware,cutlet maker, Yoda and UPX type ATM jackpotting.

+ +
Buy and sell hacked database's, accounts, logs etc
+ + +
+ + + Post Reply + + + + + + + +
+ + + + +
+
+ +
+
+
+
+ darkclonee
+ + +
Posts: 13
Joined: Sat Apr 29, 2023 10:33 am
+ + + +
+ +
+
+ +

+ Get List For WinPot malware,cutlet maker, Yoda and UPX type ATM jackpotting. +

+ + + +

+ + Post + + by darkclonee » +

+ + + +
Get List For WinPot malware,cutlet maker, Yoda and UPX type ATM jackpotting
+
+WinPot Malware
+WinPot malware
+ATMii Malware
+Cutler Malware
+
+Buy ATM jackpotting virus. WinPot requires almost no advanced knowledge or professional computer skills. The fastest way to make money.
+This software will help you bring down an ATM machine and you would be able to get the cash.
+Contact if you are interested very affordable.
+
+CONTACT
+General support wickr me...jmorgan420
+Email..... (darkclone10@proton.me)
+Telegram.....Bradley_bil7
+WhatsApp....+13239891049
+ + + + + + +
+ +
+ + + +
+
+ + + + +
+ + + Post Reply + + + + + + + + +
+ + + + + +
+ + + + +
+ +
+ +
+ + + + + + + + + + + var vglnk = { + key: 'e4fd14f5d7f2bb6d80b8f8da1354718c', + sub_id: '278e906f85e6a29e5cf5e6781f9210b0' + }; + + (function(d, t) { + var s = d.createElement(t); s.type = 'text/javascript'; s.async = true; + s.src = '//cdn.viglink.com/api/vglnk.js'; + var r = d.getElementsByTagName(t)[0]; r.parentNode.insertBefore(s, r); + }(document, 'script')); + + + + + + + + + + \ No newline at end of file diff --git a/Forums/AbyssForum/HTML_Pages/06262023/Description/viewtopicphpt849.html b/Forums/AbyssForum/HTML_Pages/06262023/Description/viewtopicphpt849.html new file mode 100644 index 0000000..a4053d7 --- /dev/null +++ b/Forums/AbyssForum/HTML_Pages/06262023/Description/viewtopicphpt849.html @@ -0,0 +1,1012 @@ + + + + + + + + + + + +BEST HACKING SERVICES - Abyss Forum| forum, drugs,guns,porn,hacking,crypto + + + + + + + + + + + + + + + + + + +{"@context":"https://schema.org","@type":"DiscussionForumPosting","url":"http://qyvjopwdgjq52ehsx6paonv2ophy3p4ivfkul4svcaw6qxlzsaboyjid.onion/viewtopic.php?t=849","headline":"BEST HACKING SERVICES","description":"We are a group of professional hackers,we are seasoned and professional hackers ...as a way of giving back to the world we are here to help with all hacking job","author":{"@type":"Person","name":"jonson"},"datePublished":"2023-03-01T01:53:09+00:00","articleSection":"Hacked Database","publisher":{"@type":"Organization","name":"Abyss Forum| forum, drugs,guns,porn,hacking,crypto","url":"http://qyvjopwdgjq52ehsx6paonv2ophy3p4ivfkul4svcaw6qxlzsaboyjid.onion"}} + + + + + + + + + + + +
+ + + + + +
+ + +

BEST HACKING SERVICES

+ +
Buy and sell hacked database's, accounts, logs etc
+ + +
+ + + Post Reply + + + + + + + +
+ + + + +
+
+ +
+
+
+
+ jonson
+ + +
Posts: 108
Joined: Fri Dec 16, 2022 11:56 pm
+ + + +
+ +
+
+ +

+ BEST HACKING SERVICES +

+ + + +

+ + Post + + by jonson » +

+ + + +
We are a group of professional hackers,we are seasoned and professional hackers ...as a way of giving back to the world we are here to help with all hacking jobs some of which are ,
+HACK INTO ANY SCHOOL DATABASE AND CHANGE UNIVERSITY GRADES, no matter
+how secured...
+What about your spouses' Cell phone? Do you need to access it anytime for location, text message data, even listen to calls?
+We can add remote access tools to networks. Install keyloggers remotely. We can also spoof calls, and depending on the situation, real time GSM interception. We offer more services such as
+PHONE HACK
+EMAIL HACK
+ALL SOCIAL MEDIAL PLATFORM HACK
+HACK INTO ANY DATABASE SYSTEM
+GRADES CHANGE HACK
+RETRIEVE LOST FILE/DOCUMENT
+ANDROID&IPHONE HACK
+ERASE CRIMINAL RECORDS
+EMAIL INTERCEPTION
+
+wickr id//; swash micheal
+
+Whatsapp: +1 (435) 527-5372
+
+Telegram:swashmichea
+ + + +
+ Last edited by jonson on Mon Jun 05, 2023 10:23 pm, edited 1 time in total. +
+ + + +
+ +
+ + + +
+
+ + +
+
+ +
+
+
+
+ jonson
+ + +
Posts: 108
Joined: Fri Dec 16, 2022 11:56 pm
+ + + +
+ +
+
+ +

+ Re: BEST HACKING SERVICES +

+ + + +

+ + Post + + by jonson » +

+ + + +
HERES MOST VERIFIED AND RELIABLE VENDOR ON MARKET WHICH OFFERS GOOD SERVICES AND DELIVERS ON TIME AND ALSO GETS EARNS YOUR TRUST HE OFFERS MOST SERVICES AND I TRIED HIM SO LETS PUT AND END TO THIS SCAMS ON MARKET AND DEAL WITH SCARFACEDARK01 I RECOMMEND HIM HE ALSO RUNS A 100% REFUNDABLE INSURANCE POLICE HERE ARE LIST OF FEW OF HIS SERVICES HE OFFERS
+SELLING WU/MU/CC/PP
+EU COVID-19 Vaccine Passport/Certificate
+SELLING RESEARCH CHEMICALS/ANABOLIC STERIODS/COKE ETC
+SELLING FIRE ARMS EXPLOSIVE /ASSASSIN SERVICES
+ALSO BEST IN HACCKING SOCIAL MEDIA ACCOUNTS/BANK ACCOUNTS/WEBSITE/BITCOIN ETC
+SELLS CRYPTO CURRENCY SOFTWARES
+AVAILABLE COUNTERFEITING BILLS EUR/AUD/USD/GBP/INR ETC
+HE OFFERS BOTH FAKE AND REAL PASSPORT/ID/DL/PERMIT/VISA ETC
+OFFERS TOP MARIJUANA CANNABIS VAPE PEN/EDIBLES
+
+
+wickr id//; swash micheal
+
+Whatsapp: +1 (435) 527-5372
+
+Telegram:swashmichea
+ + + +
+ Last edited by jonson on Mon Jun 05, 2023 10:25 pm, edited 1 time in total. +
+ + + +
+ +
+ + + +
+
+ + +
+
+ +
+
+
+
+ jonson
+ + +
Posts: 108
Joined: Fri Dec 16, 2022 11:56 pm
+ + + +
+ +
+
+ +

+ Re: BEST HACKING SERVICES +

+ + + +

+ + Post + + by jonson » +

+ + + +
We are a group of professional hackers,we are seasoned and professional hackers ...as a way of giving back to the world we are here to help with all hacking jobs some of which are ,
+HACK INTO ANY SCHOOL DATABASE AND CHANGE UNIVERSITY GRADES, no matter
+how secured...
+What about your spouses' Cell phone? Do you need to access it anytime for location, text message data, even listen to calls?
+We can add remote access tools to networks. Install keyloggers remotely. We can also spoof calls, and depending on the situation, real time GSM interception. We offer more services such as
+PHONE HACK
+EMAIL HACK
+ALL SOCIAL MEDIAL PLATFORM HACK
+HACK INTO ANY DATABASE SYSTEM
+GRADES CHANGE HACK
+RETRIEVE LOST FILE/DOCUMENT
+ANDROID&IPHONE HACK
+ERASE CRIMINAL RECORDS
+EMAIL INTERCEPTION
+
+wickr id//; swash micheal
+
+Whatsapp: +1 (435) 527-5372
+
+Telegram:swashmichea
+ + + +
+ Last edited by jonson on Mon Jun 05, 2023 10:26 pm, edited 1 time in total. +
+ + + +
+ +
+ + + +
+
+ + +
+
+ +
+
+
+
+ jonson
+ + +
Posts: 108
Joined: Fri Dec 16, 2022 11:56 pm
+ + + +
+ +
+
+ +

+ Re: BEST HACKING SERVICES +

+ + + +

+ + Post + + by jonson » +

+ + + +
HERES MOST VERIFIED AND RELIABLE VENDOR ON MARKET WHICH OFFERS GOOD SERVICES AND DELIVERS ON TIME AND ALSO GETS EARNS YOUR TRUST HE OFFERS MOST SERVICES AND I TRIED HIM SO LETS PUT AND END TO THIS SCAMS ON MARKET AND DEAL WITH SCARFACEDARK01 I RECOMMEND HIM HE ALSO RUNS A 100% REFUNDABLE INSURANCE POLICE HERE ARE LIST OF FEW OF HIS SERVICES HE OFFERS
+SELLING WU/MU/CC/PP
+EU COVID-19 Vaccine Passport/Certificate
+SELLING RESEARCH CHEMICALS/ANABOLIC STERIODS/COKE ETC
+SELLING FIRE ARMS EXPLOSIVE /ASSASSIN SERVICES
+ALSO BEST IN HACCKING SOCIAL MEDIA ACCOUNTS/BANK ACCOUNTS/WEBSITE/BITCOIN ETC
+SELLS CRYPTO CURRENCY SOFTWARES
+AVAILABLE COUNTERFEITING BILLS EUR/AUD/USD/GBP/INR ETC
+HE OFFERS BOTH FAKE AND REAL PASSPORT/ID/DL/PERMIT/VISA ETC
+OFFERS TOP MARIJUANA CANNABIS VAPE PEN/EDIBLES
+
+email:jonsonmax@protonmail.com
+wickr id//; swash micheal
+
+Whatsapp: +1 (435) 527-5372
+
+Telegram:swashmicheal
+ + + + + + +
+ +
+ + + +
+
+ + +
+
+ +
+
+
+
+ jonson
+ + +
Posts: 108
Joined: Fri Dec 16, 2022 11:56 pm
+ + + +
+ +
+
+ +

+ Re: BEST HACKING SERVICES +

+ + + +

+ + Post + + by jonson » +

+ + + +
HERES MOST VERIFIED AND RELIABLE VENDOR ON MARKET WHICH OFFERS GOOD SERVICES AND DELIVERS ON TIME AND ALSO GETS EARNS YOUR TRUST HE OFFERS MOST SERVICES AND I TRIED HIM SO LETS PUT AND END TO THIS SCAMS ON MARKET AND DEAL WITH SCARFACEDARK01 I RECOMMEND HIM HE ALSO RUNS A 100% REFUNDABLE INSURANCE POLICE HERE ARE LIST OF FEW OF HIS SERVICES HE OFFERS
+SELLING WU/MU/CC/PP
+EU COVID-19 Vaccine Passport/Certificate
+SELLING RESEARCH CHEMICALS/ANABOLIC STERIODS/COKE ETC
+SELLING FIRE ARMS EXPLOSIVE /ASSASSIN SERVICES
+ALSO BEST IN HACCKING SOCIAL MEDIA ACCOUNTS/BANK ACCOUNTS/WEBSITE/BITCOIN ETC
+SELLS CRYPTO CURRENCY SOFTWARES
+AVAILABLE COUNTERFEITING BILLS EUR/AUD/USD/GBP/INR ETC
+HE OFFERS BOTH FAKE AND REAL PASSPORT/ID/DL/PERMIT/VISA ETC
+OFFERS TOP MARIJUANA CANNABIS VAPE PEN/EDIBLES
+
+
+wickr id//; swash micheal
+
+Whatsapp: +1 (435) 527-5372
+
+Telegram:swashmichea
+ + + + + + +
+ +
+ + + +
+
+ + +
+
+ +
+
+
+
+ jonson
+ + +
Posts: 108
Joined: Fri Dec 16, 2022 11:56 pm
+ + + +
+ +
+
+ +

+ Re: BEST HACKING SERVICES +

+ + + +

+ + Post + + by jonson » +

+ + + +
HERES MOST VERIFIED AND RELIABLE VENDOR ON MARKET WHICH OFFERS GOOD SERVICES AND DELIVERS ON TIME AND ALSO GETS EARNS YOUR TRUST HE OFFERS MOST SERVICES AND I TRIED HIM SO LETS PUT AND END TO THIS SCAMS ON MARKET AND DEAL WITH SCARFACEDARK01 I RECOMMEND HIM HE ALSO RUNS A 100% REFUNDABLE INSURANCE POLICE HERE ARE LIST OF FEW OF HIS SERVICES HE OFFERS
+SELLING WU/MU/CC/PP
+EU COVID-19 Vaccine Passport/Certificate
+SELLING RESEARCH CHEMICALS/ANABOLIC STERIODS/COKE ETC
+SELLING FIRE ARMS EXPLOSIVE /ASSASSIN SERVICES
+ALSO BEST IN HACCKING SOCIAL MEDIA ACCOUNTS/BANK ACCOUNTS/WEBSITE/BITCOIN ETC
+SELLS CRYPTO CURRENCY SOFTWARES
+AVAILABLE COUNTERFEITING BILLS EUR/AUD/USD/GBP/INR ETC
+HE OFFERS BOTH FAKE AND REAL PASSPORT/ID/DL/PERMIT/VISA ETC
+OFFERS TOP MARIJUANA CANNABIS VAPE PEN/EDIBLES
+
+
+wickr id//; swash micheal
+
+Whatsapp: +1 (435) 527-5372
+
+Telegram:swashmichea
+ + + + + + +
+ +
+ + + +
+
+ + + + +
+ + + Post Reply + + + + +
+ +
+ + + + +
+ + + + + +
+ + + + +
+ +
+ +
+ + + + + + + + + + + var vglnk = { + key: 'e4fd14f5d7f2bb6d80b8f8da1354718c', + sub_id: '278e906f85e6a29e5cf5e6781f9210b0' + }; + + (function(d, t) { + var s = d.createElement(t); s.type = 'text/javascript'; s.async = true; + s.src = '//cdn.viglink.com/api/vglnk.js'; + var r = d.getElementsByTagName(t)[0]; r.parentNode.insertBefore(s, r); + }(document, 'script')); + + + + + + + + + + \ No newline at end of file diff --git a/Forums/AbyssForum/HTML_Pages/06262023/Listing/httpqyvjopwdgjq52ehsx6paonv2ophy3p4ivfkul4svcaw6qxlzsaboyjidonionviewforumphpf26sid6f7add746810784861a7ec31703a3757.html b/Forums/AbyssForum/HTML_Pages/06262023/Listing/httpqyvjopwdgjq52ehsx6paonv2ophy3p4ivfkul4svcaw6qxlzsaboyjidonionviewforumphpf26sid6f7add746810784861a7ec31703a3757.html new file mode 100644 index 0000000..b58dfa3 --- /dev/null +++ b/Forums/AbyssForum/HTML_Pages/06262023/Listing/httpqyvjopwdgjq52ehsx6paonv2ophy3p4ivfkul4svcaw6qxlzsaboyjidonionviewforumphpf26sid6f7add746810784861a7ec31703a3757.html @@ -0,0 +1,1468 @@ + + + + + + + + + + + +Hacked Database - Abyss Forum| forum, drugs,guns,porn,hacking,crypto + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + +
+ +

Hacked Database

+
+ +
Buy and sell hacked database's, accounts, logs etc
+ + + +
+ + + + New Topic + + + + + + +
+ + + + + + +
+
    +
  • +
    +
    Announcements
    +
    Replies
    +
    Views
    +
    Last post
    +
    +
  • +
+
+ +
+ +
+ + + +
+
    +
  • +
    +
    Topics
    +
    Replies
    +
    Views
    +
    Last post
    +
    +
  • +
+
+ +
+ +
+ + + +
+ + + New Topic + + + +
+ +
+ + +
+ + + + + +
+ +
+

Forum permissions

+

You cannot post new topics in this forum
You cannot reply to topics in this forum
You cannot edit your posts in this forum
You cannot delete your posts in this forum

+
+
+ +
+ + + + +
+ +
+ +
+ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Forums/AbyssForum/HTML_Pages/06262023/Listing/httpqyvjopwdgjq52ehsx6paonv2ophy3p4ivfkul4svcaw6qxlzsaboyjidonionviewforumphpf26start25.html b/Forums/AbyssForum/HTML_Pages/06262023/Listing/httpqyvjopwdgjq52ehsx6paonv2ophy3p4ivfkul4svcaw6qxlzsaboyjidonionviewforumphpf26start25.html new file mode 100644 index 0000000..106fd60 --- /dev/null +++ b/Forums/AbyssForum/HTML_Pages/06262023/Listing/httpqyvjopwdgjq52ehsx6paonv2ophy3p4ivfkul4svcaw6qxlzsaboyjidonionviewforumphpf26start25.html @@ -0,0 +1,1462 @@ + + + + + + + + + + + +Hacked Database - Page 2 - Abyss Forum| forum, drugs,guns,porn,hacking,crypto + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + +
+ +

Hacked Database

+
+ +
Buy and sell hacked database's, accounts, logs etc
+ + + +
+ + + + New Topic + + + + + + +
+ + + + + + +
+
    +
  • +
    +
    Announcements
    +
    Replies
    +
    Views
    +
    Last post
    +
    +
  • +
+
+ +
+ +
+ + + +
+
    +
  • +
    +
    Topics
    +
    Replies
    +
    Views
    +
    Last post
    +
    +
  • +
+
+ +
+ +
+ + + +
+ + + New Topic + + + +
+ +
+ + +
+ + + + + +
+ +
+

Forum permissions

+

You cannot post new topics in this forum
You cannot reply to topics in this forum
You cannot edit your posts in this forum
You cannot delete your posts in this forum

+
+
+ +
+ + + + +
+ +
+ +
+ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Forums/AbyssForum/crawler_selenium.py b/Forums/AbyssForum/crawler_selenium.py index be11fd2..573cd13 100644 --- a/Forums/AbyssForum/crawler_selenium.py +++ b/Forums/AbyssForum/crawler_selenium.py @@ -24,8 +24,6 @@ from Forums.Initialization.prepare_parser import new_parse from Forums.AbyssForum.parser import abyssForum_links_parser from Forums.Utilities.utilities import cleanHTML -config = configparser.ConfigParser() -config.read('../../setup.ini') counter = 1 baseURL = 'http://qyvjopwdgjq52ehsx6paonv2ophy3p4ivfkul4svcaw6qxlzsaboyjid.onion/' @@ -49,6 +47,8 @@ def startCrawling(): # Opens Tor Browser def opentor(): + from Forums.Initialization.forums_mining import config + global pid print("Connecting Tor...") pro = subprocess.Popen(config.get('TOR', 'firefox_binary_path')) @@ -91,6 +91,8 @@ def closetor(driver): # Creates FireFox 'driver' and configure its 'Profile' # to use Tor proxy and socket def createFFDriver(): + from Forums.Initialization.forums_mining import config + ff_binary = FirefoxBinary(config.get('TOR', 'firefox_binary_path')) ff_prof = FirefoxProfile(config.get('TOR', 'firefox_profile_path')) @@ -143,16 +145,14 @@ def savePage(page, url): # Gets the full path of the page to be saved along with its appropriate file name def getFullPathName(url): + from Forums.Initialization.forums_mining import config, CURRENT_DATE + + mainDir = os.path.join(config.get('Project', 'shared_folder'), "Forums/" + getForumName() + "/HTML_Pages") fileName = getNameFromURL(url) if isDescriptionLink(url): - #..\CryptBB\HTML_Pages\\ - fullPath = r'..\AbyssForum\HTML_Pages\\' + str( - "%02d" % date.today().month) + str("%02d" % date.today().day) + str( - "%04d" % date.today().year) + r'\\' + r'Description\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Description\\' + fileName + '.html') else: - fullPath = r'..\AbyssForum\HTML_Pages\\' + str( - "%02d" % date.today().month) + str("%02d" % date.today().day) + str( - "%04d" % date.today().year) + r'\\' + r'Listing\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Listing\\' + fileName + '.html') return fullPath @@ -170,19 +170,19 @@ def getInterestedLinks(): links = [] # Hacked Database - links.append('http://qyvjopwdgjq52ehsx6paonv2ophy3p4ivfkul4svcaw6qxlzsaboyjid.onion/viewforum.php?f=26&sid=6f7add746810784861a7ec31703a3757') + # links.append('http://qyvjopwdgjq52ehsx6paonv2ophy3p4ivfkul4svcaw6qxlzsaboyjid.onion/viewforum.php?f=26') # Hire a Hacker - # links.append('http://qyvjopwdgjq52ehsx6paonv2ophy3p4ivfkul4svcaw6qxlzsaboyjid.onion/viewforum.php?f=27&sid=6f7add746810784861a7ec31703a3757') + links.append('http://qyvjopwdgjq52ehsx6paonv2ophy3p4ivfkul4svcaw6qxlzsaboyjid.onion/viewforum.php?f=27') # # Hacking Tools - # links.append('http://qyvjopwdgjq52ehsx6paonv2ophy3p4ivfkul4svcaw6qxlzsaboyjid.onion/viewforum.php?f=28&sid=6f7add746810784861a7ec31703a3757') + # links.append('http://qyvjopwdgjq52ehsx6paonv2ophy3p4ivfkul4svcaw6qxlzsaboyjid.onion/viewforum.php?f=28') # # Carding Forums - # links.append('http://qyvjopwdgjq52ehsx6paonv2ophy3p4ivfkul4svcaw6qxlzsaboyjid.onion/viewforum.php?f=30&sid=6f7add746810784861a7ec31703a3757') + # links.append('http://qyvjopwdgjq52ehsx6paonv2ophy3p4ivfkul4svcaw6qxlzsaboyjid.onion/viewforum.php?f=30') # # Social Media Hacking - # links.append('http://qyvjopwdgjq52ehsx6paonv2ophy3p4ivfkul4svcaw6qxlzsaboyjid.onion/viewforum.php?f=32&sid=6f7add746810784861a7ec31703a3757') + # links.append('http://qyvjopwdgjq52ehsx6paonv2ophy3p4ivfkul4svcaw6qxlzsaboyjid.onion/viewforum.php?f=32') # # Hacking Tutorials - # links.append('http://qyvjopwdgjq52ehsx6paonv2ophy3p4ivfkul4svcaw6qxlzsaboyjid.onion/viewforum.php?f=12&sid=6f7add746810784861a7ec31703a3757') + # links.append('http://qyvjopwdgjq52ehsx6paonv2ophy3p4ivfkul4svcaw6qxlzsaboyjid.onion/viewforum.php?f=12') # # Cracking Tutorials - # links.append('http://qyvjopwdgjq52ehsx6paonv2ophy3p4ivfkul4svcaw6qxlzsaboyjid.onion/viewforum.php?f=13&sid=6f7add746810784861a7ec31703a3757') + # links.append('http://qyvjopwdgjq52ehsx6paonv2ophy3p4ivfkul4svcaw6qxlzsaboyjid.onion/viewforum.php?f=13') return links diff --git a/Forums/AbyssForum/parser.py b/Forums/AbyssForum/parser.py index 2b34933..5737610 100644 --- a/Forums/AbyssForum/parser.py +++ b/Forums/AbyssForum/parser.py @@ -202,139 +202,43 @@ def cryptBB_description_parser(soup): return row # This is the method to parse the Listing Pages (one page with many posts) - -def onniForums_listing_parser(soup): - - board = "-1" # board name (the previous level of the topic in the Forum categorization tree. - # For instance: Security/Malware/Tools to hack Facebook. The board here should be Malware) - - nm = 0 # this variable should receive the number of topics - topic = [] # all topics - user = [] # all users of each topic - post = [] # number of posts of each topic - view = [] # number of views of each topic - addDate = [] # when the topic was created (difficult to find) - href = [] # this variable should receive all cleaned urls (we will use this to do the marge between - # Listing and Description pages) - - # Finding the board (should be just one) - - board = soup.find('span', {"class": "active"}).text +def AbyssForums_listing_parser(soup: BeautifulSoup): + board = "-1" # board name (the previous level of the topic in the Forum categorization tree. + # For instance: Security/Malware/Tools to hack Facebook. The board here should be Malware) + + nm = 0 # this variable should receive the number of topics + topic = [] # all topics + user = [] # all users of each topic + post = [] # number of posts of each topic + view = [] # number of views of each topic + addDate = [] # when the topic was created (difficult to find) + href = [] # this variable should receive all cleaned urls (we will use this to do the marge between + # Listing and Description pages) + #finding the board + + board = soup.find("title").text board = cleanString(board.strip()) - # Finding the repeated tag that corresponds to the listing of topics + type_of_posts = soup.find_all("li", {"class": re.compile("row bg\d")} ) + for literature in type_of_posts: + title_of_post = literature.find("a", {"class": "topictitle"}).text + topic.append(title_of_post) + author = literature.find("div", {"class": "topic-poster responsive-hide left-box"}).find("a", {"class": "username"}).text + user.append(author) + num_post = literature.find("dd", {"class": "posts"}).text[1:-3] + post.append(num_post) + num_view = literature.find("dd", {"class": "views"}).text[1:-3] + view.append(num_view) + if int(num_post) != 0: + reply = literature.find("dd", {"class": "lastpost"}).find("a", {"class": "username"}).text + user.append(reply) + date_added = literature.find("time").text + addDate.append(date_added) + nm = len(topic) - itopics = soup.find_all('tr',{"class": "inline_row"}) - index = 0 - for itopic in itopics: - # For each topic found, the structure to get the rest of the information can be of two types. Testing all of them - # to don't miss any topic - # Adding the topic to the topic list - try: - topics = itopic.find('span', {"class": "subject_old"}).find('a').text - except: - topics = itopic.find('span', {"class": "subject_new"}).find('a').text - topics = re.sub("\[\w*\]", '', topics) - topic.append(cleanString(topics)) - - # Counting how many topics we have found so far - - nm = len(topic) - - # Adding the url to the list of urls - try: - link = itopic.find('span', {"class": "subject_old"}).find('a').get('href') - except: - link = itopic.find('span',{"class": "subject_new"}).find('a').get('href') - link = cleanLink(link) - href.append(link) - - # Finding the author of the topic - ps = itopic.find('div', {"class":"author smalltext"}).find('a').text - author = ps.strip() - user.append(cleanString(author)) - - # Finding the number of replies - columns = itopic.findChildren('td',recursive=False) - posts = columns[3].text - - post.append(cleanString(posts)) - - # Finding the number of Views - tview = columns[4].text - view.append(cleanString(tview)) - - - - # If no information about when the topic was added, just assign "-1" to the variable - #dt = itopic.find('div', {"class": "responsive-hide"}).text.split('»')[1] - #dt = dt.strip() - #date_time_obj = datetime.strptime(dt,'%a %b %d, %Y %I:%M %p') - #addDate.append(date_time_obj) - addDate.append("-1") - - - - index += 1 - return organizeTopics("CryptBB", nm, topic, board, view, post, user, addDate, href) - - # if len(tag) > 0: - # - # # Finding the topic - # - # tds = tds[0].find(tag[0]) - # topics = tds.text - # topics = topics.replace(u"\xbb","") - # topics = topics.strip() - # topic.append(cleanString(topics)) - # - # # Counting how many topics we have found so far - # - # nm = len(topic) - # - # # Adding the url to the list of urls - # - # link = tds.findAll('a', href=True) - # link = link[0].get('href') - # link = cleanLink(link) - # href.append(link) - # - # # Finding the author of the topic - # - # ps = itopic.find('td', {"class": tag[1]}).find('p').find('a') - # if ps == None: - # ps = itopic.find('td', {"class": tag[1]}).find('p') - # ps = ps.text.replace("Started by ","") - # else: - # ps = ps.text - # author = ps.strip() - # user.append(cleanString(author)) - # - # # Finding the number of replies - # - # statistics = itopic.find('td', {"class": tag[2]}) - # statistics = statistics.text - # statistics = statistics.split("Replies") - # posts = statistics[0].strip() - # post.append(cleanString(posts)) - # - # # Finding the number of Views - # - # views = statistics[1] - # views = views.replace("Views","") - # views = views.strip() - # view.append(cleanString(views)) - # - # # As no information about when the topic was added, just assign "-1" to the variable - # - # addDate.append("-1") - - #return organizeTopics("TheMajesticGarden", nm, topic, board, view, post, user, addDate, href) - -#need to change this method def abyssForum_links_parser(soup): # Returning all links that should be visited by the Crawler diff --git a/Forums/Altenens/crawler_selenium.py b/Forums/Altenens/crawler_selenium.py index 9670014..547b765 100644 --- a/Forums/Altenens/crawler_selenium.py +++ b/Forums/Altenens/crawler_selenium.py @@ -24,8 +24,6 @@ from Forums.Initialization.prepare_parser import new_parse from Forums.Altenens.parser import altenens_links_parser from Forums.Utilities.utilities import cleanHTML -config = configparser.ConfigParser() -config.read('../../setup.ini') counter = 1 baseURL = 'https://altenens.is/' @@ -49,6 +47,8 @@ def startCrawling(): # Opens Tor Browser def opentor(): + from Forums.Initialization.forums_mining import config + global pid print("Connecting Tor...") pro = subprocess.Popen(config.get('TOR', 'firefox_binary_path')) @@ -61,18 +61,16 @@ def opentor(): # Login using premade account credentials and do login captcha manually def login(driver): #click login button - login = driver.find_element(by=By.XPATH, value='/html/body/div[1]/div[1]/div/div/div/div[1]/a[1]').\ - get_attribute('href') - driver.get(login) - # login.click() - - # #entering username and password into input boxes - # usernameBox = driver.find_element(by=By.XPATH, value='/html/body/div[1]/div[4]/div/div/div[3]/div/div/div/form/div[1]/div/dl[1]/dd') - # #Username here - # usernameBox.send_keys('mylittlepony45')#sends string to the username box - # passwordBox = driver.find_element(by=By.XPATH, value='/html/body/div[1]/div[4]/div/div/div[3]/div/div/div/form/div[1]/div/dl[2]/dd/div/div') - # #Password here - # passwordBox.send_keys('johnnyTest@18')# sends string to passwordBox + login = driver.find_element(by=By.XPATH, value='//*[@id="top"]/div[1]/div/div/div/div[1]/a[1]') + login.click() + + #entering username and password into input boxes + usernameBox = driver.find_element(by=By.XPATH, value='//*[@id="_xfUid-1-1688066635"]') + #Username here + usernameBox.send_keys('mylittlepony45')#sends string to the username box + passwordBox = driver.find_element(by=By.XPATH, value='//*[@id="_xfUid-2-1688066635"]') + #Password here + passwordBox.send_keys('johnnyTest@18')# sends string to passwordBox input("Press ENTER when CAPTCHA is completed\n") @@ -108,6 +106,8 @@ def closetor(driver): # Creates FireFox 'driver' and configure its 'Profile' # to use Tor proxy and socket def createFFDriver(): + from Forums.Initialization.forums_mining import config + ff_binary = FirefoxBinary(config.get('TOR', 'firefox_binary_path')) ff_prof = FirefoxProfile(config.get('TOR', 'firefox_profile_path')) @@ -160,15 +160,14 @@ def savePage(page, url): # Gets the full path of the page to be saved along with its appropriate file name def getFullPathName(url): + from Forums.Initialization.forums_mining import config, CURRENT_DATE + + mainDir = os.path.join(config.get('Project', 'shared_folder'), "Forums/" + getForumName() + "/HTML_Pages") fileName = getNameFromURL(url) if isDescriptionLink(url): - fullPath = r'..\\Altenens\\HTML_Pages\\' + str( - "%02d" % date.today().month) + str("%02d" % date.today().day) + str( - "%04d" % date.today().year) + r'\\' + r'Description\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Description\\' + fileName + '.html') else: - fullPath = r'..\\Altenens\\HTML_Pages\\' + str( - "%02d" % date.today().month) + str("%02d" % date.today().day) + str( - "%04d" % date.today().year) + r'\\' + r'Listing\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Listing\\' + fileName + '.html') return fullPath diff --git a/Forums/CryptBB/HTML_Pages/06262023/Description/Read/showthreadphptid16584.html b/Forums/CryptBB/HTML_Pages/06262023/Description/Read/showthreadphptid16584.html new file mode 100644 index 0000000..8afc5b5 --- /dev/null +++ b/Forums/CryptBB/HTML_Pages/06262023/Description/Read/showthreadphptid16584.html @@ -0,0 +1,28 @@ + Buy registered passports, drivers license, ID card.Telegram.....(@dndocsforsale) + +

Buy registered passports, drivers license, ID card.Telegram.....(@dndocsforsale)
#1
Buy registered passports, drivers license, ID card and certificates, Covid-19 vaccine card,


Telegram................................ (@dndocsforsale)


Emaill.......................................dndocsforsaleonline@protonmail.com


We are the best producers of HIGH QUALITY document,Getting a fake and a real (genuine) passport, ID or driving license or any other document is simple. we can make you both real and fake documents.

However, the real documents are more e??ensive than the fake because

it takes time, skill and contacts to get it done. Note that, the fake

is going to be 100% unique and in very good quality. The difference is based on the registration of the numbers. The real Document will be registered with the country's database so you can use it to travel to any country of your choice or in the country, mean while the fake will not be registered but can be used as well.


Buy fake USA,EU,UK,AU,Canada and fake Israeli passports,


Telegram................................ (@dndocsforsale)


Emaill.......................................dndocsforsaleonline@protonmail.com

We provide Covid-19 vaccine card/Certificate/ passport


Buy USA Covid-19 card (registered under your state department of health )


Buy Canada Covid-19 card (Registered on the Healthcare )


Buy UK NHS covid-19 pass ( Registered under NHS system)


Buy Ireland (Registered with the HSE system)


Buy EU digital covid-19 certificate for European countries

Buy Australian Covid19 Digital Certificates




=OUR SERVICES


Buy CSCS Cards


Buy Passport and ID Cards


Buy Diplomatic passports


Buy Birth Certificates


Buy other country documents


Buy Drivers License SSN


Buy Citizenship Documents


Buy Marriage and Divorce certificates


Buy High school certificates and Degree


Buy TOEIC, TOEFL, IELTS, GRE Score & Certificate


Telegram................................ (@dndocsforsale)


Emaill.......................................dndocsforsaleonline@protonmail.com
Reply Report



+ // + + $(".author_avatar img").error(function () { + $(this).unbind("error").closest('.author_avatar').remove(); + }); +
\ No newline at end of file diff --git a/Forums/CryptBB/HTML_Pages/06262023/Description/Read/showthreadphptid2628.html b/Forums/CryptBB/HTML_Pages/06262023/Description/Read/showthreadphptid2628.html new file mode 100644 index 0000000..c9504b8 --- /dev/null +++ b/Forums/CryptBB/HTML_Pages/06262023/Description/Read/showthreadphptid2628.html @@ -0,0 +1,28 @@ + Python begineer Guide + +

Python begineer Guide
#1
Python Beginner Guide

----------------------------------------------------------------------

it a great choice for beginners and experts alike. Python's readability makes it a great first programming language — it allows you to think like a programmer and not waste time with confusing syntax. ... If Python isn't installed, it's easy to download and install. That's why i write this tutorial
Introduction

Python is a interpreted, object-oriented, high-level programming language.
Python was created by Guido van Rossum and officially released in the year 1991.
This language is still quite new and is one of the easier languages to learn.
Another really great thing about python is that unlike many programming languages, it does NOT require a semi-colon(";") at the end of every line.

• Python comes pre-installed with Mac OS X and
Linux.
• Windows binaries from http://python.org/
• You might not have to do anything!
Rich, built-in collection types:
Lists
Tuples
Dictionaries (maps)
Sets


How to Install Python on Windows

How to Install Python IDE ?
Below is a step by step process on how to download and install Python on Windows

1) To download and install Python, visit the official website of Python https://www.python.org/downloads/ and choose your version.

2) Once the download is completed, run the .exe file to install Python. Now click on Install Now.

3) You can see Python installing at this point .




Good example of scripting language
“Pythonic” style is very concise
Powerful but unobtrusive object system
Every value is an object
Powerful collection and iteration
abstractions
Dynamic typing makes generics easy
simplified syntax and not complicated,
which gives more emphasis on natural language.
It is used in web development, data science, creating software prototypes and so on.
Fortunately for beginners, Python has simple easy-to-use syntax. This makes Python an excellent language to learn to program for beginners

Table of contents
  • Getting started
  • Variables
  • Operators
  • Logical Statements
  • Lists
  • Loops
  • Functions
  • Modules
  • Global Variables
  • Final words & Credits




----------------------------------------------------------------------



Getting started



I recommend you download and install Python, since Python 2.7 is outdated.just download Python 3

If you want to write your own Python programs, you need to use an IDE many begineers dont know full name of ide so full name is = integrated Development Environment
There are a lot of different IDE's for python, personally i prefer PyCharm.

If you are not yet familiar with python, i recommend you use JetBrains PyCharm Edu, which you can download for free here.
This version of PyCharm offers you a diversity of tutorials, aswell as extra help features.


If you are familiar with python already, i recommend you use JetBrains Pycharm Community Edition, which you can download for free here.
This version of PyCharm offers you alot more plugins and features.


In python you can add a comment to your code using the hash symbol ('#').
Example:
Code:
#This is a comment!

If you wish to have multiple lines of comments, you can enclose the lines of comments using the double quote symbol 3 times.
Example:
Code:
"""
These
Are
Multiple
Lines
Of
Comments
"""

Comments will always be ignored by the interpreter and are usually used to add some notes to a part of the code. (if u complete 1 project and you have to work after 2 months or long time that notes will helping to understand how you create that things )



----------------------------------------------------------------------

Python TUPLE - Pack, Unpack , Compare, Slicing of Tuple , Tuples and dictionary , tuples as keys in dictionaries

What is Tuple Matching in Python?

-Tuple Matching in Python is a method of grouping the tuples by matching the second element in the tuples. It is achieved by using a dictionary by checking the second element in each tuple in python programming. However, we can make new tuples by taking portions of existing tuples.

Tuple Syntax

Code:
Tup = ('Jan','feb','march')

To write an empty tuple = tup1 = ();

For writing tuple for a single value, you need to include a comma like = Tup1 = (10,);

Tuple indices begin at 0, and they can be concatenated, sliced and so on

Packing and Unpacking :

In packing, we place value into a new tuple while in unpacking we extract those values back into variables

Code:
x = ("xyz99", 20, "imm")    # tuple packing
(company, emp, profile) = x    # tuple unpacking
print(company)
print(emp)
print(profile)

Comparing tuples

comparison operator in Python can work with tuples
comparison starts with a first element of each tuple. If they do not compare to =,< or > then it proceed to the second element and so on
#case 1
Code:
a=(5,6)
b=(1,4)
if (a>b):print("a is bigger")
else: print("b is bigger")
#case 2
Code:
a=(5,6)
b=(5,4)
if (a>b):print("a is bigger")
else: print ("b is bigger")

Case1: Comparison starts with a first element of each tuple. In this case 5>1, so the output a is bigger

Case 2: Comparison starts with a first element of each tuple. In this case 5>5 which is inconclusive. So it proceeds to the next element. 6>4, so the output a is bigger

tuples as keys in dictionaries

tuple is a hashable value and can be used as a dictionary key. A tuple would be useful as a key when storing values associated with a
grid or some other coordinate type system.

Example: We would come across a composite key if we need to create a telephone directory that maps, first-name, last-name, pairs of telephone numbers, etc. Assuming that we have declared the variables as last and first number, we could write a dictionary assignment statement as shown below:
Code:
directory[last,first] = number or other value or string
Inside the brackets, the expression is a tuple. We could use tuple assignment in a for loop to navigate this dictionary.
Code:
for last, first in directory:
    print first, last, directory[last, first]
This loop navigates the keys in the directory, which are tuples. It assigns the elements of each tuple to last and first and then prints the name and corresponding telephone number.

Tuples and dictionary
Dictionary can return the list of tuples by calling items, with each tuple is a key value pair
Code:
a = {'x':100, 'y':200}
b = list(a.items())
print(b)


Deleting Tuples

Tuples are immutable and cannot be deleted. You cannot delete or remove items from a tuple. But deleting tuple entirely is possible by using the keyword = ' del '

Slicing of Tuple
To fetch specific sets of sub-elements from tuple or list, we use this unique function called slicing. its also used for array and list.

Code:
x = ("x", "y","z", "f", "g")
print(x[2:4])
The output of this code will be ('z', 'f').

----------------------------------------------------------------------
Variables

In python, variables are what you use to store values.
The three most important variables in python are:
  • Integer
An integer (int) is a variable where you can store a number that can be written without a fractional component.

check blank box for check content or go to google and search about it
Spoiler:


a = 70
b = -50
c = 0


  • Float
A float is a variable where you can store a number that is written with a fractional component.

Spoiler:


a = 80.42
b = -50.99
c = 0.000


  • String
An string is a variable where you can store characters if you enclose them in quotes.
Python treats single quotes the same as double quotes.
Example:
Spoiler:

a = "Hello"
b = "World"
c = "Cyberjagu"



----------------------------------------------------------------------


Operators


Python has a lot of different operators, such as mathematicial operators and comparison(logical) operators.

Here is a list of some of the mathematical operators python uses.

  • +
This operator adds the value's on both sides of the operator.
Spoiler:

a + b = 20


  • -
This operator substracts the value on the right side of the operator from the value on the left side of the operator.
Spoiler:

a - b = 10


  • *
This operator multiplies the values of on both sides of the operator.
Spoiler:

a * b = 100


  • /
This operator divides the value on the right side of the operator by the value on the left side of the operator.
Spoiler:

a / b = 1



  • %
This operator divides the value on the right side of the operator by the value on the left side of the operator and returns the remainder.
Spoiler:

a % b = 0



Here is a list of some of the comparison operators python uses.
These are the operators you will be using when you are working with while statements and for loops etc.


  • ==
If the value of both operands are equal, then the condition becomes true.
Spoiler:

if a == b:
print("a is equal to b")


  • !=
If the value of both operands are not equal, then the condition becomes true.
Spoiler:

if a != b:
print("a is not equal to b")


  • >
If the value of the left operand is greater then the value of the right operand, then the condition becomes true.
Spoiler:

if a > b:
print("a is bigger then b")


  • <
If the value of the left operand is less then the value of the right operand, then the condition becomes true.
Spoiler:

if a < b:
print("a is smaller then b")


  • >=
If the value of the left operand is equal to or bigger then the value of the right operand, then the condition becomes true.
Spoiler:

if a >= b:
print("a is equal or bigger then b")


  • <=
If the value of the left operand is equal or less then the value of the right operand, then the condition becomes true.
Spoiler:

if a <= b:
print("a is equal or smaller then b")




----------------------------------------------------------------------


Logical Statements


Logical statements are also known as if/else statements.
They are statements that check whether the conditions are true or false.
In these statements we are going to use the comparison operators we just discussed.

A common mistake i made when i started out with Python was forgetting to add the ':' after the If statement line.
Don't forget to add this, if you don't the statement won't work and you will get an _error_.

Example:
Code:
a = 10
b = 10

if a == b:
    print("a is equal to b")
else:
    print("a is not equal to b")

If you run this code it will print "a is equal to b", because both variables are 10.


Example of a slightly more complicated if statement:
Code:
a = 10
b = 10
c = 20
isittrue = True

if a == c:
    isittrue = True
elif a == b:
    isittrue = True
else:
    print("A is not equal to b or c")
    isittrue = False

if a == b or a == c:
    isittrue = True
else:
    print("A is not equal to b or c")
    isittrue = False

if a == b and isittrue == True:
    c = c - b
    print(c)


The first If statement checks if a is equal to c or if a is equal to b. Since a is equal to b, the variable isittrue will be set to True.
While this is a correct code, programmers are lazy people and always strive to make their code as short as possible.

That's why the second If statement checks exactly the same as the first if statement, but uses the or operator instead.
This way the or operator saves 2 lines of code, which in this case shouldn't matter alot, but if you are coding a big program, efficient coding will save you alot of time.

The third If statement checks if a is equal to b and if the variable isittrue is set on True.
If this is both the case (which it is), c will be set to be c - b.
Basicly what happens is the variable c is now 20 - 10.
After that the print statement will show 10 as the new value of the variable c.


----------------------------------------------------------------------


Lists

Lists are a very important part of python.
A list is a variable that holds multiple values. A list in Python is similar to an array in Java.
You can declare a list the same as any other variable, except that it needs to be enclosed with brackets ("[]") and seperated with commas(",").

It's very important to remember that lists work with positions.
The first position is always 0, NOT 1

Example:
Code:
list = ["One", "Two", "Three"]
print (list)
print(list[0])
print(list[1])
print(list[2])

When you run this code, you will get this output:

Code:
["One", "Two", "Three"]
One
Two
Three

You might wonder what just happened, well:
Print (list) tells python to print everything that is in the list, which is why the commas and brackets get printed aswell.
Print (list[0] tells python to print the first variable in the variables list.
Print (list[1] tells python to print the second variable in the variables list.
Print (list[2] tells python to print the third variable in the variables list.

You might wonder why or how this could be usefull. I will get back to that later in this guide.


----------------------------------------------------------------------


Loops

In Python there are several kinds of loops.
Right now we will discuss the two most important ones.


While loop

This is one of the easiest and most basic loops of python. What a while loop does, is perfor a series of events as long as something is in a certain condition.
With the while loop you also need to add the colon sign after the line (':')

Example:
Code:
a = 5

while a > 0:
    print (a)
    a = a - 1
print("We are out of the loop!")
As you can see first we declare a variable, and then the while loop checks if the variable is bigger then 0.
This while loop will keep running the events within it until a is equal to 0.
So your output will be:

Code:
5
4
3
2
1
We are out of the loop!

You can use all comparison operators in a while loop to get the result you want.


For loop

A for loop is used for iterating over a sequence (that is either a list, a tuple, a dictionary, a set, or a string).
This is less like the for keyword in other programming languages, and works more like an iterator method as found in other object-orientated programming languages.
With the for loop we can execute a set of statements, once for each item in a list, tuple, set etc.
The for loop is slightly more complex then the while loop, but it's still perfectly understandable.
The for loop executes a sequence of statements multiple times.
With the while loop you also need to add the colon sign after the line (':')

Let me explain this loop with an example.
Code:
for x in range(5):
    print (x)
       Or
fruits = ["apple", "banana", "cherry"]
for x in fruits:
  print(x)
(try it your self :D)


If you run this code your output will be:

Code:
0
1
2
3
4

The for loop goes through every item in a list before it can exit the loop.
In this example "x" is a variable. It does not have a set value, because it's used to run the for loop.

In simple terms: for x in a list with a range of 5, print x

Now you might be wondering, why does it stop at 4, when we said range 5?
You have to remember, that a for loop works with lists, and as mentioned before, the first position of a list is 0.



----------------------------------------------------------------------


Functions

Functions are very important when you are coding a program or script.
A function is in simple terms a specific piece of code.
A function is very efficient when you need to run a function multiple times, for example in a loop.

Here a simple example of a define:
Code:
def definitionname():
    print("This is my define.")
definitionname()

The "def" is how you start the definition. After that you put in a unique name for your definition. This can be anything you like.
With the define you also need to add the colon sign after the line (':')

If you want to run the code within the define, you can call it by typing the name of the function with parameters ("()")
The output for the code above would be: This is my define.


As mentioned before a function is especially efficient when you want to run the code in the function multiple times.

For example:
Code:
def multipletimes():
    print ("My very first define.")

for x in range(5):
    multipletimes()

When you run this code your output will be:

Code:
My very first define.
My very first define.
My very first define.
My very first define.
My very first define.


This is a very simple example, but you can already see how efficient it is.
This way the print statement is executed 5 times, with only 4 lines of code.
When you are coding a complex program or script defines will make your job a lot easier!


----------------------------------------------------------------------


Modules

Modules are python programs that you can use.
These mudules are scripts written by another person in order for you to be able to make a fairly complex script/program, without extensive knowledge about the functions you are using.

To be able to use a module, you first have to import it. You can simply do this by typing 'import' and the Python module name.

For example:
Code:
import time


Time is a basic Python module that is included in the standard python package you downloaded at the start of this thread.
You are now able to use this module.

A simple example of how you can use the time module:
Code:
import time
seconds = time.time()

print("number of seconds since 10:00am, Januari 1 1970:", seconds)

When you run this code the output will be something like:
number of ticks since 10:00am, Januari 1 1970: 1457813005.3186388

You are probably thinking: How can i know every single module name and every statement within a module?
The answer to that is simple. Google what you need! For example, if you want to make a running clock, you can google: Python clock.

In the results you will most likely find a site such as stackoverflow. On this site it will probably tell you that in order for you to make a clock in python, you need to import the time module.


----------------------------------------------------------------------


Global Variables

At last i want to explain what global variables are and what you can use them for.
First of all, try to avoid global variables as much as possible! With big scripts it would get really messy really quik.

That being said, a global variable is a normal variable, not unlike the normal variables we discussed earlier in this thread.
The difference between normal variables and global variables is that global variables are being used within loops and statements.

For example:

Code:
x = 10

def normalx():
    x = 5
normalx()

print (x)

def globalx():
    global x
    x = 20
globalx()

print (x)

In this example we declare the variable x as 10.
Then we run the function normalx, and after the function has been ran we print x.
Then we run the function globalx, and after the function has been ran we print x again.

When you run this code your output will be:
Code:
10
20


The reason for this is that in the first define, x is not called as the global x.
In the second define we do call x as global x, which is why the global x variable has been changed from 10 to 20.

recommendations


Whenever you are stuck, either look up https://tutorialspoint.com and go to the python tutorial, or look for solutions on stackoverflow. Smart googling will most likely yield stackoverflow results whenever python is mentioned.

For applying and honing your programming skills:
check out these python modules and play around with them a little
For install them
Use = pip forwindows or pip3 for linux or mac
Open your terminal ==> pip/pip3 install sockets(modual name)


socket
socks
selenium
pyppeteer
requests

many users apply for Full Membership (i think its usefull for them I can't write everything related to challange This is just a Guideline
I am not even tell everything will be done by using this, I am just giving you an idea )


the first two are modules for internet and OS internal communications via a concept called socket. all internet communication is based upon sockets and their operations (except for wifis and some layers of LANs). If you can master sockets, then you can build basic web servers or other services, like building your own communication networks, botnets, whatever needs communicatin'
socks is a socket-esque module that supports socks4 and socks5 proxies, this is very useful for communication via tor.


selenium and pyppeteer are two modules that allow you to automate a web browser, allowing you to automatically click buttons, fill out forms and so on. you can use this for creating mass email accounts, writing social media comments or, and this is my main metier, trading bots (or at least it used to be, way back in the day before I caught wind of trading APIs), I for instance wrote a bot that logged into and controlled btc accounts on it's own using selenium. these complex tasks are a bit hard to do with sockets or requests, because most web apps are highly complex and use hundreds of requests and forms for things that just appear to be one single button.

requests, the online python communities favorite, is a module to automate http requests. that's it.


there are also various cryptography libraries out there featuring the current algorithms. feel free to write your own password manager as a simple CLI


Many moduals are available Just search it (google is your Friend)

----------------------------------------------------------------------


Final words & Credits

Congratulations! You have finished this python tutorial!
Right now you know the basics of Python.

Right now you are thinking: Hmm alright it was fun and all, but what do i do now with my knowledge?
Simple! Practice with Python. You can code a variety of programs and scripts, from a logger to an email bomber to battlefield 2.

Ofcourse coding in python still requires some effort, but if you ever run into trouble, feel free to shoot me a PM.

I will always be available to answer all questions regarding Python.


Credits:

Some Online research and some forum


If you have found any mistakes in this tutorial, please notify me and i will edit it as soon as possible.
Happy programming!

You know that I have worked hard in this, so a good comment of yours will help me to post similar tutorials.
Cyberjagu.
Cyberjagu@jabber.calyxinstitute.org [OTR]
Reply Report
#2
Nice , it will be good for newbies
Amateurs hack systems, Professionals hack people.
Reply Report
#3
sorry to ask is it from hack forums [.]net?
Reply Report
#4
(10-29-2020, 02:00 PM)theQ Wrote: sorry to ask is it from hack forums [.]net?

Yes you are right this format from hackforums i will added more topic and information in future (you can do that also )
you can see i add many things here (like some colors lines using color tags , some codes loop , some begineer topics too )
Cyberjagu@jabber.calyxinstitute.org [OTR]
Reply Report
#5
(10-29-2020, 02:02 PM)cyberjagu Wrote:
(10-29-2020, 02:00 PM)theQ Wrote: sorry to ask is it from hack forums [.]net?

Yes you are right this format from hackforums i will added more topic and information in future (you can do that also )
you can see i add many things here

Yep, you have added a lot of additional information brother, I am really glad for this post .
Reply Report
#6
(10-29-2020, 02:06 PM)theQ Wrote:
(10-29-2020, 02:02 PM)cyberjagu Wrote:
(10-29-2020, 02:00 PM)theQ Wrote: sorry to ask is it from hack forums [.]net?

Yes you are right this format from hackforums i will added more topic and information in future (you can do that also )
you can see i add many things here

Yep, you have added a lot of additional information brother, I am really glad for this post .

you can suggest me topics so i can improve this thread
Cyberjagu@jabber.calyxinstitute.org [OTR]
Reply Report
#7
(10-29-2020, 02:07 PM)cyberjagu Wrote: you can suggest me topics so i can improve this thread
OSINT, rat/ransomware for educational purposes brother. I guess there are not lot of information on OSINT, so it would be helpful if you make one brother.
Reply Report
#8
(10-29-2020, 02:10 PM)theQ Wrote:
(10-29-2020, 02:07 PM)cyberjagu Wrote: you can suggest me topics so i can improve this thread
OSINT, rat/ransomware for educational purposes brother. I guess there are not lot of information on OSINT, so it would be helpful if you make one brother.

its good idea but i will add it later because i want to cover all basic and intermidiate informations I'll do it when the thread is perfect and ready to go advanced
Cyberjagu@jabber.calyxinstitute.org [OTR]
Reply Report
#9
(10-29-2020, 02:12 PM)cyberjagu Wrote:
(10-29-2020, 02:10 PM)theQ Wrote:
(10-29-2020, 02:07 PM)cyberjagu Wrote: you can suggest me topics so i can improve this thread
OSINT, rat/ransomware for educational purposes brother. I guess there are not lot of information on OSINT, so it would be helpful if you make one brother.

its good idea but i will add it later because i want to cover all basic and intermidiate informations I'll do it when the thread is perfect and ready to go advanced

Thanks brother, If you have leaked courses of OSINT please share for time being and also make a thread behind the logic of rat/ransomware preferably in python {if you are free and available brother}.
Reply Report
#10
one of the best thread i had ever seen
Reply Report



+ // + + $(".author_avatar img").error(function () { + $(this).unbind("error").closest('.author_avatar').remove(); + }); +
\ No newline at end of file diff --git a/Forums/CryptBB/HTML_Pages/06262023/Listing/httpcryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebidonionforumdisplayphpfid86.html b/Forums/CryptBB/HTML_Pages/06262023/Listing/httpcryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebidonionforumdisplayphpfid86.html new file mode 100644 index 0000000..f2d510c --- /dev/null +++ b/Forums/CryptBB/HTML_Pages/06262023/Listing/httpcryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebidonionforumdisplayphpfid86.html @@ -0,0 +1,8 @@ + CryptBB - Beginner Programming + +


Rules
1. Read the sticky before posting.

Beginner Programming
Thread / Author Replies Views Last Post [asc]
Important Threads
  34 37,452 04-08-2023, 08:48 AM
Last Post: alphenzo
  0 16,561 12-24-2019, 03:24 PM
Last Post: Power
Normal Threads
  0 3 39 minutes ago
Last Post: Moneytimes22
  0 13 4 hours ago
Last Post: aloy567
  0 29 Today, 05:57 AM
Last Post: MRBORO
  0 54 Yesterday, 04:42 AM
Last Post: Darkclone
  0 53 Yesterday, 03:48 AM
Last Post: Darkclone
  0 58 06-24-2023, 07:46 PM
Last Post: thugisland
  1 263 06-22-2023, 12:57 AM
Last Post: CULTIST
  0 81 06-21-2023, 09:18 PM
Last Post: surecake
  2 328 06-21-2023, 03:55 PM
Last Post: Xprogrammer
  3 320 06-20-2023, 10:00 PM
Last Post: OliviaDavis77
  0 97 06-20-2023, 09:37 AM
Last Post: lodkfjfff
  0 120 06-19-2023, 10:35 PM
Last Post: Dadian
  0 130 06-19-2023, 05:44 PM
Last Post: give-me-head-until-im-dead
  0 123 06-19-2023, 08:40 AM
Last Post: codfjfjff
  0 129 06-18-2023, 11:19 PM
Last Post: Realgeneraldocs
  0 127 06-18-2023, 11:18 PM
Last Post: Realgeneraldocs
  0 165 06-16-2023, 02:14 PM
Last Post: ANKUSH-IDFC@
  0 192 06-13-2023, 10:46 PM
Last Post: Realitydocs




\ No newline at end of file diff --git a/Forums/CryptBB/HTML_Pages/06262023/Listing/httpcryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebidonionforumdisplayphpfid86page2.html b/Forums/CryptBB/HTML_Pages/06262023/Listing/httpcryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebidonionforumdisplayphpfid86page2.html new file mode 100644 index 0000000..e474eb3 --- /dev/null +++ b/Forums/CryptBB/HTML_Pages/06262023/Listing/httpcryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebidonionforumdisplayphpfid86page2.html @@ -0,0 +1,8 @@ + CryptBB - Beginner Programming + +


Rules
1. Read the sticky before posting.

Beginner Programming
Thread / Author Replies Views Last Post [asc]
  0 187 06-13-2023, 10:45 PM
Last Post: Realitydocs
  0 177 06-13-2023, 10:42 PM
Last Post: Realitydocs
  2 644 06-13-2023, 06:58 PM
Last Post: mistaken4kings
 
Best IDE? (Pages: 1 2 3 4 ... 7 )
60 53,641 06-13-2023, 06:54 PM
Last Post: mistaken4kings
  3 1,010 06-13-2023, 06:51 PM
Last Post: mistaken4kings
  0 158 06-11-2023, 10:14 PM
Last Post: Invader99@
  4 827 06-09-2023, 10:38 PM
Last Post: ColdReloaded
  0 279 06-06-2023, 10:20 AM
Last Post: PROCTOR07
  4 955 06-06-2023, 09:37 AM
Last Post: primer
  2 867 06-05-2023, 11:11 PM
Last Post: open-unit
  1 633 06-05-2023, 11:07 PM
Last Post: open-unit
  4 807 05-31-2023, 07:11 PM
Last Post: phosphate
  4 1,073 05-31-2023, 08:28 AM
Last Post: softech
  1 800 05-29-2023, 07:15 PM
Last Post: Username 1
  3 2,396 05-25-2023, 09:23 PM
Last Post: clueless
  6 1,264 05-17-2023, 06:28 AM
Last Post: Username 1
  4 1,328 05-15-2023, 04:11 PM
Last Post: elbeastz
  8 1,556 05-15-2023, 11:02 AM
Last Post: spidermonkey
  0 754 05-12-2023, 08:13 PM
Last Post: Aristotle
  2 1,165 05-12-2023, 07:56 AM
Last Post: yany3




\ No newline at end of file diff --git a/Forums/CryptBB/crawler_selenium.py b/Forums/CryptBB/crawler_selenium.py index d9a4766..d432647 100644 --- a/Forums/CryptBB/crawler_selenium.py +++ b/Forums/CryptBB/crawler_selenium.py @@ -17,37 +17,36 @@ from PIL import Image import urllib.parse as urlparse import os, re, time import subprocess -import configparser from bs4 import BeautifulSoup from Forums.Initialization.prepare_parser import new_parse from Forums.CryptBB.parser import cryptBB_links_parser from Forums.Utilities.utilities import cleanHTML -config = configparser.ConfigParser() -config.read('../../setup.ini') counter = 1 baseURL = 'http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/' # Opens Tor Browser, crawls the website def startCrawling(): - opentor() + # opentor() forumName = getForumName() - driver = getAccess() - - if driver != 'down': - try: - login(driver) - crawlForum(driver) - except Exception as e: - print(driver.current_url, e) - closetor(driver) + # driver = getAccess() + # + # if driver != 'down': + # try: + # login(driver) + # crawlForum(driver) + # except Exception as e: + # print(driver.current_url, e) + # closetor(driver) new_parse(forumName, baseURL, False) # Opens Tor Browser def opentor(): + from Forums.Initialization.forums_mining import config + global pid print("Connecting Tor...") pro = subprocess.Popen(config.get('TOR', 'firefox_binary_path')) @@ -133,6 +132,8 @@ def closetor(driver): # Creates FireFox 'driver' and configure its 'Profile' # to use Tor proxy and socket def createFFDriver(): + from Forums.Initialization.forums_mining import config + ff_binary = FirefoxBinary(config.get('TOR', 'firefox_binary_path')) ff_prof = FirefoxProfile(config.get('TOR', 'firefox_profile_path')) @@ -186,12 +187,14 @@ def savePage(page, url): # Gets the full path of the page to be saved along with its appropriate file name def getFullPathName(url): - from Forums.Initialization.forums_mining import CURRENT_DATE + from Forums.Initialization.forums_mining import config, CURRENT_DATE + + mainDir = os.path.join(config.get('Project', 'shared_folder'), "Forums/" + getForumName() + "/HTML_Pages") fileName = getNameFromURL(url) if isDescriptionLink(url): - fullPath = r'..\\CryptBB\\HTML_Pages\\' + CURRENT_DATE + r'\\Description\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Description\\' + fileName + '.html') else: - fullPath = r'..\\CryptBB\\HTML_Pages\\' + CURRENT_DATE + r'\\Listing\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Listing\\' + fileName + '.html') return fullPath @@ -264,7 +267,7 @@ def crawlForum(driver): driver.refresh() savePage(driver.page_source, item) driver.back() - ''' + #variable to check if there is a next page for the topic has_next_topic_page = True counter = 1 @@ -287,16 +290,16 @@ def crawlForum(driver): if item == "": raise NoSuchElementException - has_next_topic_page = False else: counter += 1 + except NoSuchElementException: has_next_topic_page = False # end of loop for i in range(counter): driver.back() - ''' + # comment out break diff --git a/Forums/DB_Connection/db_connection.py b/Forums/DB_Connection/db_connection.py index eeaf69b..340abfc 100644 --- a/Forums/DB_Connection/db_connection.py +++ b/Forums/DB_Connection/db_connection.py @@ -9,8 +9,8 @@ def connectDataBase(): try: - config = configparser.ConfigParser() - config.read('../../setup.ini') + from Forums.Initialization.forums_mining import config + ip = config.get('PostgreSQL', 'ip') username = config.get('PostgreSQL', 'username') password = config.get('PostgreSQL', 'password') diff --git a/Forums/HiddenAnswers/crawler_selenium.py b/Forums/HiddenAnswers/crawler_selenium.py index 34f3c07..66085a3 100644 --- a/Forums/HiddenAnswers/crawler_selenium.py +++ b/Forums/HiddenAnswers/crawler_selenium.py @@ -24,8 +24,6 @@ from Forums.Initialization.prepare_parser import new_parse from Forums.HiddenAnswers.parser import hiddenanswers_links_parser from Forums.Utilities.utilities import cleanHTML -config = configparser.ConfigParser() -config.read('../../setup.ini') counter = 1 baseURL = 'http://7eoz4h2nvw4zlr7gvlbutinqqpm546f5egswax54az6lt2u7e3t6d7yd.onion/' @@ -49,6 +47,8 @@ def startCrawling(): # Opens Tor Browser def opentor(): + from Forums.Initialization.forums_mining import config + global pid print("Connecting Tor...") pro = subprocess.Popen(config.get('TOR', 'firefox_binary_path')) @@ -91,6 +91,8 @@ def closetor(driver): # Creates FireFox 'driver' and configure its 'Profile' # to use Tor proxy and socket def createFFDriver(): + from Forums.Initialization.forums_mining import config + ff_binary = FirefoxBinary(config.get('TOR', 'firefox_binary_path')) ff_prof = FirefoxProfile(config.get('TOR', 'firefox_profile_path')) @@ -143,16 +145,14 @@ def savePage(page, url): # Gets the full path of the page to be saved along with its appropriate file name def getFullPathName(url): + from Forums.Initialization.forums_mining import config, CURRENT_DATE + + mainDir = os.path.join(config.get('Project', 'shared_folder'), "Forums/" + getForumName() + "/HTML_Pages") fileName = getNameFromURL(url) if isDescriptionLink(url): - #..\CryptBB\HTML_Pages\\ - fullPath = r'..\HiddenAnswers\HTML_Pages\\' + str( - "%02d" % date.today().month) + str("%02d" % date.today().day) + str( - "%04d" % date.today().year) + r'\\' + r'Description\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Description\\' + fileName + '.html') else: - fullPath = r'..\HiddenAnswers\HTML_Pages\\' + str( - "%02d" % date.today().month) + str("%02d" % date.today().day) + str( - "%04d" % date.today().year) + r'\\' + r'Listing\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Listing\\' + fileName + '.html') return fullPath diff --git a/Forums/Initialization/forumsList.txt b/Forums/Initialization/forumsList.txt index 3b57198..2c8a141 100644 --- a/Forums/Initialization/forumsList.txt +++ b/Forums/Initialization/forumsList.txt @@ -1 +1,2 @@ OnniForums +CryptBB \ No newline at end of file diff --git a/Forums/Initialization/forums_mining.py b/Forums/Initialization/forums_mining.py index 5e8aa92..da42857 100644 --- a/Forums/Initialization/forums_mining.py +++ b/Forums/Initialization/forums_mining.py @@ -13,12 +13,18 @@ from Forums.AbyssForum.crawler_selenium import crawler as crawlerAbyssForum from Forums.Altenens.crawler_selenium import crawler as crawlerAltenensForum from Forums.HiddenAnswers.crawler_selenium import crawler as crawlerHiddenAnswers +<<<<<<< HEAD +======= +import configparser +>>>>>>> 98de158ee6f14b35b2a5ac61427d7de36b4ee442 import time +config = configparser.ConfigParser() +config.read('../../setup.ini') CURRENT_DATE = str("%02d" % date.today().month) + str("%02d" % date.today().day) + str("%04d" % date.today().year) @@ -37,7 +43,8 @@ def createDirectory(forum): if forum == 'Reddits': pagesMainDir = '../' + forum else: - pagesMainDir = '../' + forum + "/HTML_Pages" + # pagesMainDir = '../' + forum + "/HTML_Pages" + pagesMainDir = os.path.join(config.get('Project', 'shared_folder'), "Forums/" + forum + "/HTML_Pages") if not os.path.isdir(pagesMainDir): os.makedirs(pagesMainDir) diff --git a/Forums/Initialization/geckodriver.log b/Forums/Initialization/geckodriver.log index 5f040ff..0d1d5ba 100644 --- a/Forums/Initialization/geckodriver.log +++ b/Forums/Initialization/geckodriver.log @@ -6224,8 +6224,13 @@ unwatchForTargets()@TargetList.jsm:37 destructor()@TargetList.jsm:109 stop()@CDP.jsm:104 close()@RemoteAgent.jsm:138 +<<<<<<< HEAD 1687895546413 geckodriver INFO Listening on 127.0.0.1:52237 1687895550932 mozrunner::runner INFO Running command: "C:\\Users\\\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "52238" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofileiOR21Q" +======= +1687896430885 geckodriver INFO Listening on 127.0.0.1:50135 +1687896434527 mozrunner::runner INFO Running command: "C:\\Users\\calsyslab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" ... "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofilenQCzgp" +>>>>>>> 98de158ee6f14b35b2a5ac61427d7de36b4ee442 console.log: "TorSettings: loadFromPrefs()" console.log: "TorConnect: init()" console.log: "TorConnect: Entering Initial state" @@ -6234,7 +6239,7 @@ console.log: "TorConnect: Observing topic 'TorProcessExited'" console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'" console.log: "TorConnect: Observing topic 'torsettings:ready'" console.log: "TorSettings: Observed profile-after-change" -1687895551675 Marionette INFO Marionette enabled +1687896435185 Marionette INFO Marionette enabled console.log: "TorConnect: Will load after bootstrap => [about:blank]" console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. @@ -6242,19 +6247,64 @@ JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't fin JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +<<<<<<< HEAD DevTools listening on ws://localhost:52238/devtools/browser/ad1dc524-5cad-4983-9dd6-c7f6f3d5caee 1687895553974 Marionette INFO Listening on port 52243 1687895554561 RemoteAgent WARN TLS certificate errors will be ignored for this session JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofileiOR21Q\thumbnails) because it does not exist 1687895804567 Marionette INFO Stopped listening on port 52243 +======= +DevTools listening on ws://localhost:50136/devtools/browser/773adaec-44e1-4b13-9fac-c38bfb170221 +1687896436579 Marionette INFO Listening on port 50142 +1687896436612 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\calsyslab\AppData\Local\Temp\rust_mozprofilenQCzgp\thumbnails) because it does not exist +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/, line 2: ReferenceError: $ is not defined +1687896481968 Marionette INFO Stopped listening on port 50142 +>>>>>>> 98de158ee6f14b35b2a5ac61427d7de36b4ee442 JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] !!! error running onStopped callback: TypeError: callback is not a function JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first. -JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofileiOR21Q\thumbnails) because it does not exist +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\calsyslab\AppData\Local\Temp\rust_mozprofilenQCzgp\thumbnails) because it does not exist + +###!!! [Child][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + ###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost -1687895804907 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +1687896482482 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1687897304511 geckodriver INFO Listening on 127.0.0.1:50201 +1687897308111 mozrunner::runner INFO Running command: "C:\\Users\\calsyslab\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" ... "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\CALSYS~1\\AppData\\Local\\Temp\\rust_mozprofile2TNTj7" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: init()" +console.log: "TorConnect: Entering Initial state" +console.log: "TorConnect: Observed profile-after-change" +console.log: "TorConnect: Observing topic 'TorProcessExited'" +console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: Observing topic 'torsettings:ready'" +console.log: "TorSettings: Observed profile-after-change" +1687897308686 Marionette INFO Marionette enabled +console.log: "TorConnect: Will load after bootstrap => [about:blank]" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:50202/devtools/browser/c30256b0-c71f-40da-a95f-bb1313b3e35e +1687897310328 Marionette INFO Listening on port 50208 +1687897310788 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/, line 2: ReferenceError: $ is not defined +1687897315273 Marionette INFO Stopped listening on port 50208 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first. +JavaScript error: resource://gre/modules/PageThumbs.jsm, line 709: AbortError: IOUtils.profileBeforeChange getter: IOUtils: profileBeforeChange phase has already finished +1687897315776 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 unwatchForTabs()@TargetList.jsm:70 unwatchForTargets()@TargetList.jsm:37 destructor()@TargetList.jsm:109 diff --git a/Forums/Initialization/prepare_parser.py b/Forums/Initialization/prepare_parser.py index 668a86d..b062129 100644 --- a/Forums/Initialization/prepare_parser.py +++ b/Forums/Initialization/prepare_parser.py @@ -2,7 +2,7 @@ __author__ = 'DarkWeb' import codecs import glob -import os +import os, re import shutil from Forums.DB_Connection.db_connection import * from Forums.BestCardingWorld.parser import * @@ -80,7 +80,7 @@ def persist_data(url, row, cur): #calls the different parser methods here depending on the type of html page def new_parse(forum, url, createLog): - from Forums.Initialization.forums_mining import CURRENT_DATE + from Forums.Initialization.forums_mining import config, CURRENT_DATE print("Parsing The " + forum + " Forum and conduct data classification to store the information in the database.") @@ -97,7 +97,8 @@ def new_parse(forum, url, createLog): lines = [] # listing pages lns = [] # description pages - detPage = {} + detPage = {} # first pages + other = {} # other pages # Creating the log file for each Forum if createLog: @@ -109,12 +110,14 @@ def new_parse(forum, url, createLog): " in the _Logs folder to read files from this Forum of this date again.") raise SystemExit + mainDir = os.path.join(config.get('Project', 'shared_folder'), "Forums/" + forum + "/HTML_Pages") + # Reading the Listing Html Pages - for fileListing in glob.glob(os.path.join("..\\" + forum + "\\HTML_Pages\\" + CURRENT_DATE + "\\Listing", '*.html')): + for fileListing in glob.glob(os.path.join(mainDir, CURRENT_DATE + "\\Listing", '*.html')): lines.append(fileListing) # Reading the Description Html Pages - for fileDescription in glob.glob(os.path.join("..\\" + forum + "\\HTML_Pages\\" + CURRENT_DATE + "\\Description" ,'*.html')): + for fileDescription in glob.glob(os.path.join(mainDir, CURRENT_DATE + "\\Description", '*.html')): lns.append(fileDescription) # Parsing the Description Pages and put the tag's content into a dictionary (Hash table) @@ -153,8 +156,17 @@ def new_parse(forum, url, createLog): # key = u"Top:" + rmm[0].upper().strip() + u" User:" + rmm[2][0].upper().strip() key = u"Url:" + os.path.basename(line2).replace(".html", "") - # save file address with description record in memory - detPage[key] = {'rmm': rmm, 'filename': os.path.basename(line2)} + # check if page or page exists at the end of a string followed by a series of numbers + #if yes add to other if no add to first page dictionary + # save descritions into record in memory + check = re.compile(r'(?<=Page|page)[0-9]*') + if check.search(key): + # print(key, 'is an other page\n') + other[key] = {'rmm': rmm, 'filename': os.path.basename(line2)} + else: + # print(key, 'is a first page\n') + detPage[key] = {'rmm': rmm, 'files': [os.path.basename(line2)]} + except: @@ -163,6 +175,27 @@ def new_parse(forum, url, createLog): if createLog: logFile.write(str(nError) + ". There was a problem to parse the file " + line2 + " in the Description section.\n") + # goes through keys from detPage and other, checks if the keys match. + # if yes adds other[key] values to detPage w/o overwritting + for key in detPage.keys(): + for k in list(other.keys()): + checkkey = str(key[4:]) + checkk = str(k[4:]) + + if checkkey in checkk: + detPage[key]['rmm'][1].extend(other[k]['rmm'][1]) + detPage[key]['rmm'][2].extend(other[k]['rmm'][2]) + detPage[key]['rmm'][3].extend(other[k]['rmm'][3]) + detPage[key]['rmm'][4].extend(other[k]['rmm'][4]) + detPage[key]['rmm'][5].extend(other[k]['rmm'][5]) + detPage[key]['rmm'][6].extend(other[k]['rmm'][6]) + detPage[key]['rmm'][7].extend(other[k]['rmm'][7]) + detPage[key]['rmm'][8].extend(other[k]['rmm'][8]) + detPage[key]['files'].append(other[k]['filename']) + + other.pop(k) + + # Parsing the Listing Pages and put the tag's content into a list for index, line1 in enumerate(lines): @@ -259,20 +292,21 @@ def new_parse(forum, url, createLog): if not persistError: # move description files of completed folder - source = line2.replace(os.path.basename(line2), "") + detPage[key]['filename'] - destination = line2.replace(os.path.basename(line2), "") + r'Read/' - - try: - shutil.move(source, destination) - num_persisted_moved += 1 - except: - - print("There was a problem to move the file " + detPage[key]['filename'] + " in the Description section!") - nError += 1 - if createLog: - logFile.write( - str(nError) + ". There was a problem to move the file " + detPage[key]['filename'] + " in the Description section!.\n") - moveError = True + for filename in detPage[key]['files']: + source = line2.replace(os.path.basename(line2), "") + filename + destination = line2.replace(os.path.basename(line2), "") + r'Read/' + + try: + shutil.move(source, destination) + num_persisted_moved += 1 + except: + + print("There was a problem to move the file " + filename + " in the Description section!") + nError += 1 + if createLog: + logFile.write( + str(nError) + ". There was a problem to move the file " + filename + " in the Description section!.\n") + moveError = True # if the associated description page is not read or not parsed else: diff --git a/Forums/OnniForums/HTML_Pages/06272023/Listing/httponnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qdonionForumHackingCrackingtutorials.html b/Forums/OnniForums/HTML_Pages/06272023/Listing/httponnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qdonionForumHackingCrackingtutorials.html index 03de7d8..3dc3822 100644 --- a/Forums/OnniForums/HTML_Pages/06272023/Listing/httponnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qdonionForumHackingCrackingtutorials.html +++ b/Forums/OnniForums/HTML_Pages/06272023/Listing/httponnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qdonionForumHackingCrackingtutorials.html @@ -1,4 +1,8 @@ +<<<<<<< HEAD OnniForums - Hacking & Cracking tutorials (1) +======= + OnniForums - Hacking & Cracking tutorials +>>>>>>> 98de158ee6f14b35b2a5ac61427d7de36b4ee442 var unreadAlerts = '1'; +======= + var my_post_key = "6491120fbe576a317267f2d3c4eceb5a"; + +// --> +>>>>>>> 98de158ee6f14b35b2a5ac61427d7de36b4ee442 -
You have one unread private message from dkota titled Welcome to the Forum!

Users browsing this forum: Molnjar, purely_cabbage
Hacking & Cracking tutorials
Thread / Author Replies Views Last Post [asc]
Forum Announcements
   
How to buy Rank Upgrades!
- - 01-16-2023, 02:09 AM
   
Hacked 2 skid forums using 0-days
- - 01-13-2023, 12:29 PM
Normal Threads
   
HOW TO GET INF STORAGE ON MEGA (Pages: 1 2 3 4 ... 8 )
CRACKY34, 04-17-2023, 03:20 AM
70 696
5 minutes ago
Last Post: Molnjar
   
12 239
1 hour ago
Last Post: fatfed
 
[Exclusive] RCE on MyBB forums with Magic Byte (Pages: 1 2 3 4 ... 9 )
prince97, 04-17-2023, 01:41 AM
83 882
1 hour ago
Last Post: fatfed
 
Big Learning Collection (Pages: 1 2 3 4 ... 37 )
Levron, 03-17-2023, 06:09 PM
368 2,910
1 hour ago
Last Post: WMIHRIG
 
notes which i often use for "pentesting" + bonus (Pages: 1 2 3 4 ... 20 )
linsley, 03-28-2023, 12:13 PM
196 1,772
3 hours ago
Last Post: Mor3AmaD3uz
 
TCM Security Courses (Pages: 1 2 3 4 ... 9 )
Levron, 03-17-2023, 06:02 PM
84 1,070
3 hours ago
Last Post: nexialist1
 
Hack botnet panels (Pages: 1 2 3 4 ... 9 )
prince97, 02-12-2023, 07:44 PM
89 1,179
5 hours ago
Last Post: mskop
   
203 2,255
Yesterday, 05:47 PM
Last Post: Ezraco
 
Python Fundamentals
demoted, 06-18-2023, 01:19 AM
0 52
06-18-2023, 01:19 AM
Last Post: demoted
 
How to start hacking - part 1
dkota, 06-16-2023, 04:17 PM
4 114
06-18-2023, 12:41 AM
Last Post: zonefour
   
4 58
06-18-2023, 12:41 AM
Last Post: TheRealZeroSec
   
Private ChatGPT
Mr.Robot, 05-31-2023, 10:58 AM
1 159
06-16-2023, 01:03 AM
Last Post: blackmonkey2523
   
Jailbreak ChatGPT
prince97, 03-17-2023, 02:15 PM
4 562
06-14-2023, 11:06 PM
Last Post: OptimusCrime
   
Kali Linux 2023.2 is out
OptimusCrime, 06-02-2023, 04:07 PM
6 85
06-14-2023, 04:32 PM
Last Post: user1234567
   
3 63
06-12-2023, 05:39 PM
Last Post: big_boobies
   
OpenSSL 101
DerRoteMilan, 06-08-2023, 06:56 AM
0 32
06-08-2023, 06:56 AM
Last Post: DerRoteMilan
   
All about SSH
OptimusCrime, 06-02-2023, 04:15 PM
1 61
06-02-2023, 07:41 PM
Last Post: Funshine
 
BUNCH of RAT's for FREE
nemesis, 03-11-2023, 12:27 AM
7 548
06-02-2023, 12:14 AM
Last Post: Nohrix
   
Speech synthesis- chatgpt
Mr.Robot, 05-31-2023, 11:18 AM
1 77
05-31-2023, 01:27 PM
Last Post: Spiral
   
4 303
05-28-2023, 12:57 PM
Last Post: DeSnake


  New Posts
  Hot Thread (New)
  Hot Thread (No New)
  No New Posts
  Contains Posts by You
  Closed Thread


\ No newline at end of file +<<<<<<< HEAD +
You have one unread private message from dkota titled Welcome to the Forum!

Users browsing this forum: Molnjar, purely_cabbage
Hacking & Cracking tutorials
Thread / Author Replies Views Last Post [asc]
Forum Announcements
   
How to buy Rank Upgrades!
- - 01-16-2023, 02:09 AM
   
Hacked 2 skid forums using 0-days
- - 01-13-2023, 12:29 PM
Normal Threads
   
HOW TO GET INF STORAGE ON MEGA (Pages: 1 2 3 4 ... 8 )
CRACKY34, 04-17-2023, 03:20 AM
70 696
5 minutes ago
Last Post: Molnjar
   
12 239
1 hour ago
Last Post: fatfed
 
[Exclusive] RCE on MyBB forums with Magic Byte (Pages: 1 2 3 4 ... 9 )
prince97, 04-17-2023, 01:41 AM
83 882
1 hour ago
Last Post: fatfed
 
Big Learning Collection (Pages: 1 2 3 4 ... 37 )
Levron, 03-17-2023, 06:09 PM
368 2,910
1 hour ago
Last Post: WMIHRIG
 
notes which i often use for "pentesting" + bonus (Pages: 1 2 3 4 ... 20 )
linsley, 03-28-2023, 12:13 PM
196 1,772
3 hours ago
Last Post: Mor3AmaD3uz
 
TCM Security Courses (Pages: 1 2 3 4 ... 9 )
Levron, 03-17-2023, 06:02 PM
84 1,070
3 hours ago
Last Post: nexialist1
 
Hack botnet panels (Pages: 1 2 3 4 ... 9 )
prince97, 02-12-2023, 07:44 PM
89 1,179
5 hours ago
Last Post: mskop
   
203 2,255
Yesterday, 05:47 PM
Last Post: Ezraco
 
Python Fundamentals
demoted, 06-18-2023, 01:19 AM
0 52
06-18-2023, 01:19 AM
Last Post: demoted
 
How to start hacking - part 1
dkota, 06-16-2023, 04:17 PM
4 114
06-18-2023, 12:41 AM
Last Post: zonefour
   
4 58
06-18-2023, 12:41 AM
Last Post: TheRealZeroSec
   
Private ChatGPT
Mr.Robot, 05-31-2023, 10:58 AM
1 159
06-16-2023, 01:03 AM
Last Post: blackmonkey2523
   
Jailbreak ChatGPT
prince97, 03-17-2023, 02:15 PM
4 562
06-14-2023, 11:06 PM
Last Post: OptimusCrime
   
Kali Linux 2023.2 is out
OptimusCrime, 06-02-2023, 04:07 PM
6 85
06-14-2023, 04:32 PM
Last Post: user1234567
   
3 63
06-12-2023, 05:39 PM
Last Post: big_boobies
   
OpenSSL 101
DerRoteMilan, 06-08-2023, 06:56 AM
0 32
06-08-2023, 06:56 AM
Last Post: DerRoteMilan
   
All about SSH
OptimusCrime, 06-02-2023, 04:15 PM
1 61
06-02-2023, 07:41 PM
Last Post: Funshine
 
BUNCH of RAT's for FREE
nemesis, 03-11-2023, 12:27 AM
7 548
06-02-2023, 12:14 AM
Last Post: Nohrix
   
Speech synthesis- chatgpt
Mr.Robot, 05-31-2023, 11:18 AM
1 77
05-31-2023, 01:27 PM
Last Post: Spiral
   
4 303
05-28-2023, 12:57 PM
Last Post: DeSnake


  New Posts
  Hot Thread (New)
  Hot Thread (No New)
  No New Posts
  Contains Posts by You
  Closed Thread


+======= +
You have one unread private message from dkota titled Welcome to the Forum!

Users browsing this forum: purely_cabbage, Zerohours, 2 Invisible User(s)
Hacking & Cracking tutorials
Thread / Author Replies Views Last Post [asc]
Forum Announcements
   
How to buy Rank Upgrades!
- - 01-16-2023, 02:09 AM
   
New privacy XMPP/Jabber server
- - 01-13-2023, 12:29 AM
Normal Threads
   
7 168
50 minutes ago
Last Post: cyberiagu
   
HOW TO GET INF STORAGE ON MEGA (Pages: 1 2 3 4 ... 11 )
CRACKY34, 04-17-2023, 03:20 AM
104 976
1 hour ago
Last Post: Huggywuggy
 
notes which i often use for "pentesting" + bonus (Pages: 1 2 3 4 ... 27 )
linsley, 03-28-2023, 12:13 PM
263 2,195
1 hour ago
Last Post: jecoso3008
 
Big Learning Collection (Pages: 1 2 3 4 ... 46 )
Levron, 03-17-2023, 06:09 PM
455 3,492
1 hour ago
Last Post: Rainbows
 
[Exclusive] RCE on MyBB forums with Magic Byte (Pages: 1 2 3 4 ... 15 )
prince97, 04-17-2023, 01:41 AM
149 1,327
1 hour ago
Last Post: pX1_9u
 
TCM Security Courses (Pages: 1 2 3 4 ... 13 )
Levron, 03-17-2023, 06:02 PM
120 1,330
1 hour ago
Last Post: pX1_9u
   
289 2,763
1 hour ago
Last Post: xrc
   
5 111
4 hours ago
Last Post: alcapone
   
1 22
5 hours ago
Last Post: ruski
 
Hack botnet panels (Pages: 1 2 3 4 ... 15 )
prince97, 02-12-2023, 07:44 PM
144 1,544
8 hours ago
Last Post: fafa654
 
How to start hacking - part 1 (Pages: 1 2 )
dkota, 06-16-2023, 04:17 PM
10 368
10 hours ago
Last Post: Kuromi
   
Help with OSINT website (Pages: 1 2 3 4 5 )
somekindofwizard, 06-24-2023, 03:22 AM
43 222
Yesterday, 12:03 PM
Last Post: lilyamongus
 
Newbie
Diamonq, Yesterday, 02:00 AM
1 33
Yesterday, 02:06 AM
Last Post: Sneaky
   
HackTheBox :: Pilgrimage Discussion/Solution (Pages: 1 2 )
Helghast, 06-26-2023, 09:14 PM
10 95
06-26-2023, 10:51 PM
Last Post: Helghast
   
shitcord
yummyv2, 06-26-2023, 12:10 PM
0 51
06-26-2023, 12:10 PM
Last Post: yummyv2
 
BUNCH of RAT's for FREE
nemesis, 03-11-2023, 12:27 AM
8 639
06-26-2023, 10:38 AM
Last Post: dubLEDrE
 
0 54
06-26-2023, 06:59 AM
Last Post: vipper
   
14 269
06-25-2023, 11:17 PM
Last Post: slash
   
0 53
06-25-2023, 09:50 AM
Last Post: mangeky023
 
Python Fundamentals
demoted, 06-18-2023, 01:19 AM
1 104
06-25-2023, 05:46 AM
Last Post: bornLosing


  New Posts
  Hot Thread (New)
  Hot Thread (No New)
  No New Posts
  Contains Posts by You
  Closed Thread

Search this Forum:


+>>>>>>> 98de158ee6f14b35b2a5ac61427d7de36b4ee442 diff --git a/Forums/OnniForums/crawler_selenium.py b/Forums/OnniForums/crawler_selenium.py index 5a61e4f..2c32ef0 100644 --- a/Forums/OnniForums/crawler_selenium.py +++ b/Forums/OnniForums/crawler_selenium.py @@ -25,8 +25,6 @@ from Forums.Initialization.prepare_parser import new_parse from Forums.OnniForums.parser import onniForums_links_parser from Forums.Utilities.utilities import cleanHTML -config = configparser.ConfigParser() -config.read('../../setup.ini') counter = 1 baseURL = 'http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/' @@ -50,6 +48,8 @@ def startCrawling(): # Opens Tor Browser def opentor(): + from Forums.Initialization.forums_mining import config + global pid print("Connecting Tor...") pro = subprocess.Popen(config.get('TOR', 'firefox_binary_path')) @@ -109,6 +109,8 @@ def closetor(driver): # Creates FireFox 'driver' and configure its 'Profile' # to use Tor proxy and socket def createFFDriver(): + from Forums.Initialization.forums_mining import config + ff_binary = FirefoxBinary(config.get('TOR', 'firefox_binary_path')) ff_prof = FirefoxProfile(config.get('TOR', 'firefox_profile_path')) @@ -163,16 +165,14 @@ def savePage(page, url): # Gets the full path of the page to be saved along with its appropriate file name def getFullPathName(url): + from Forums.Initialization.forums_mining import config, CURRENT_DATE + + mainDir = os.path.join(config.get('Project', 'shared_folder'), "Forums/" + getForumName() + "/HTML_Pages") fileName = getNameFromURL(url) if isDescriptionLink(url): - #..\CryptBB\HTML_Pages\\ - fullPath = r'..\OnniForums\HTML_Pages\\' + str( - "%02d" % date.today().month) + str("%02d" % date.today().day) + str( - "%04d" % date.today().year) + r'\\' + r'Description\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Description\\' + fileName + '.html') else: - fullPath = r'..\OnniForums\HTML_Pages\\' + str( - "%02d" % date.today().month) + str("%02d" % date.today().day) + str( - "%04d" % date.today().year) + r'\\' + r'Listing\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Listing\\' + fileName + '.html') return fullPath @@ -189,10 +189,10 @@ def getNameFromURL(url): def getInterestedLinks(): links = [] - # Hacking & Cracking tutorials - links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Hacking-Cracking-tutorials') + # # Hacking & Cracking tutorials + # links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Hacking-Cracking-tutorials') # Hacking & Cracking questions - # links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Hacking-Cracking-questions') + links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Hacking-Cracking-questions') # # Exploit PoCs # links.append('http://onnii6niq53gv3rvjpi7z5axkasurk2x5w5lwliep4qyeb2azagxn4qd.onion/Forum-Exploit-PoCs') # # Cracked software @@ -279,15 +279,13 @@ def crawlForum(driver): for i in range(counter): driver.back() - - # comment out, one topic per page break # comment out, go through all pages if count == 1: - count = 0 - break + count = 0 + break try: temp = driver.find_element(by=By.XPATH, value= diff --git a/MarketPlaces/AnonymousMarketplace/crawler_selenium.py b/MarketPlaces/AnonymousMarketplace/crawler_selenium.py index a37915c..dd35a69 100644 --- a/MarketPlaces/AnonymousMarketplace/crawler_selenium.py +++ b/MarketPlaces/AnonymousMarketplace/crawler_selenium.py @@ -25,8 +25,6 @@ from MarketPlaces.Initialization.prepare_parser import new_parse from MarketPlaces.AnonymousMarketplace.parser import anonymous_links_parser from MarketPlaces.Utilities.utilities import cleanHTML -config = configparser.ConfigParser() -config.read('../../setup.ini') counter = 1 baseURL = 'http://3fqr7fgjaslhgmeiin5e2ky6ra5xkiafyzg7i36sfcehv3jvpgydteqd.onion/' @@ -52,6 +50,8 @@ def startCrawling(): # Opens Tor Browser #prompts for ENTER input to continue def opentor(): + from MarketPlaces.Initialization.markets_mining import config + global pid print("Connecting Tor...") pro = subprocess.Popen(config.get('TOR', 'firefox_binary_path')) @@ -90,6 +90,8 @@ def closetor(driver): # Creates FireFox 'driver' and configure its 'Profile' # to use Tor proxy and socket def createFFDriver(): + from MarketPlaces.Initialization.markets_mining import config + ff_binary = FirefoxBinary(config.get('TOR', 'firefox_binary_path')) ff_prof = FirefoxProfile(config.get('TOR', 'firefox_profile_path')) @@ -154,12 +156,14 @@ def savePage(page, url): # Gets the full path of the page to be saved along with its appropriate file name #@param: raw url as crawler crawls through every site def getFullPathName(url): - from MarketPlaces.Initialization.markets_mining import CURRENT_DATE + from MarketPlaces.Initialization.markets_mining import config, CURRENT_DATE + + mainDir = os.path.join(config.get('Project', 'shared_folder'), "MarketPlaces/" + getMKTName() + "/HTML_Pages") fileName = getNameFromURL(url) if isDescriptionLink(url): - fullPath = r'..\AnonymousMarketplace\HTML_Pages\\' + CURRENT_DATE + r'\\Description\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Description\\' + fileName + '.html') else: - fullPath = r'..\AnonymousMarketplace\HTML_Pages\\' + CURRENT_DATE + r'\\Listing\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Listing\\' + fileName + '.html') return fullPath diff --git a/MarketPlaces/Apocalypse/crawler_selenium.py b/MarketPlaces/Apocalypse/crawler_selenium.py index d01814d..b46e4af 100644 --- a/MarketPlaces/Apocalypse/crawler_selenium.py +++ b/MarketPlaces/Apocalypse/crawler_selenium.py @@ -25,8 +25,6 @@ from MarketPlaces.Initialization.prepare_parser import new_parse from MarketPlaces.Apocalypse.parser import apocalypse_links_parser from MarketPlaces.Utilities.utilities import cleanHTML -config = configparser.ConfigParser() -config.read('../../setup.ini') counter = 1 baseURL = 'http://apocam5hnoqskkmhr325nivjuh5phbmmggadxgcjabzzirap5iklkxad.onion/' @@ -52,6 +50,8 @@ def startCrawling(): # Opens Tor Browser #prompts for ENTER input to continue def opentor(): + from MarketPlaces.Initialization.markets_mining import config + global pid print("Connecting Tor...") pro = subprocess.Popen(config.get('TOR', 'firefox_binary_path')) @@ -90,6 +90,8 @@ def closetor(driver): # Creates FireFox 'driver' and configure its 'Profile' # to use Tor proxy and socket def createFFDriver(): + from MarketPlaces.Initialization.markets_mining import config + ff_binary = FirefoxBinary(config.get('TOR', 'firefox_binary_path')) ff_prof = FirefoxProfile(config.get('TOR', 'firefox_profile_path')) @@ -170,12 +172,14 @@ def savePage(page, url): # Gets the full path of the page to be saved along with its appropriate file name #@param: raw url as crawler crawls through every site def getFullPathName(url): - from MarketPlaces.Initialization.markets_mining import CURRENT_DATE + from MarketPlaces.Initialization.markets_mining import config, CURRENT_DATE + + mainDir = os.path.join(config.get('Project', 'shared_folder'), "MarketPlaces/" + getMKTName() + "/HTML_Pages") fileName = getNameFromURL(url) if isDescriptionLink(url): - fullPath = r'..\Apocalypse\HTML_Pages\\' + CURRENT_DATE + r'\\Description\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Description\\' + fileName + '.html') else: - fullPath = r'..\Apocalypse\HTML_Pages\\' + CURRENT_DATE + r'\\Listing\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Listing\\' + fileName + '.html') return fullPath diff --git a/MarketPlaces/CityMarket/crawler_selenium.py b/MarketPlaces/CityMarket/crawler_selenium.py index c1c3e88..1384c18 100644 --- a/MarketPlaces/CityMarket/crawler_selenium.py +++ b/MarketPlaces/CityMarket/crawler_selenium.py @@ -26,8 +26,6 @@ from MarketPlaces.Initialization.prepare_parser import new_parse from MarketPlaces.CityMarket.parser import city_links_parser from MarketPlaces.Utilities.utilities import cleanHTML -config = configparser.ConfigParser() -config.read('../../setup.ini') counter = 1 baseURL = 'http://wsptlnuoo3johqzcdlwuj5zcwfh2dwmswz6hahqctuxttvxpanypmwad.onion/' @@ -53,6 +51,8 @@ def startCrawling(): # Opens Tor Browser #prompts for ENTER input to continue def opentor(): + from MarketPlaces.Initialization.markets_mining import config + global pid print("Connecting Tor...") pro = subprocess.Popen(config.get('TOR', 'firefox_binary_path')) @@ -91,6 +91,8 @@ def closetor(driver): # Creates FireFox 'driver' and configure its 'Profile' # to use Tor proxy and socket def createFFDriver(): + from MarketPlaces.Initialization.markets_mining import config + ff_binary = FirefoxBinary(config.get('TOR', 'firefox_binary_path')) ff_prof = FirefoxProfile(config.get('TOR', 'firefox_profile_path')) @@ -167,12 +169,14 @@ def savePage(page, url): # Gets the full path of the page to be saved along with its appropriate file name #@param: raw url as crawler crawls through every site def getFullPathName(url): - from MarketPlaces.Initialization.markets_mining import CURRENT_DATE + from MarketPlaces.Initialization.markets_mining import config, CURRENT_DATE + + mainDir = os.path.join(config.get('Project', 'shared_folder'), "MarketPlaces/" + getMKTName() + "/HTML_Pages") fileName = getNameFromURL(url) if isDescriptionLink(url): - fullPath = r'..\CityMarket\HTML_Pages\\' + CURRENT_DATE + r'\\Description\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Description\\' + fileName + '.html') else: - fullPath = r'..\CityMarket\HTML_Pages\\' + CURRENT_DATE + r'\\Listing\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Listing\\' + fileName + '.html') return fullPath diff --git a/MarketPlaces/CypherMarketplace/crawler_selenium.py b/MarketPlaces/CypherMarketplace/crawler_selenium.py index 3b758ce..aa587c4 100644 --- a/MarketPlaces/CypherMarketplace/crawler_selenium.py +++ b/MarketPlaces/CypherMarketplace/crawler_selenium.py @@ -2,6 +2,7 @@ __author__ = 'Helium' ''' CypherMarketplace Forum Crawler (Selenium) +crawler done ''' from selenium import webdriver @@ -24,8 +25,6 @@ from MarketPlaces.Initialization.prepare_parser import new_parse from MarketPlaces.CypherMarketplace.parser import cyphermarketplace_links_parser from MarketPlaces.Utilities.utilities import cleanHTML -config = configparser.ConfigParser() -config.read('../../setup.ini') counter = 1 baseURL = 'http://6c5qa2ke2esh6ake6u6yoxjungz2czbbl7hqxl75v5k37frtzhxuk7ad.onion/' @@ -51,6 +50,8 @@ def startCrawling(): # Opens Tor Browser #prompts for ENTER input to continue def opentor(): + from MarketPlaces.Initialization.markets_mining import config + global pid print("Connecting Tor...") pro = subprocess.Popen(config.get('TOR', 'firefox_binary_path')) @@ -89,6 +90,8 @@ def closetor(driver): # Creates FireFox 'driver' and configure its 'Profile' # to use Tor proxy and socket def createFFDriver(): + from MarketPlaces.Initialization.markets_mining import config + ff_binary = FirefoxBinary(config.get('TOR', 'firefox_binary_path')) ff_prof = FirefoxProfile(config.get('TOR', 'firefox_profile_path')) @@ -167,12 +170,14 @@ def savePage(page, url): # Gets the full path of the page to be saved along with its appropriate file name #@param: raw url as crawler crawls through every site def getFullPathName(url): - from MarketPlaces.Initialization.markets_mining import CURRENT_DATE + from MarketPlaces.Initialization.markets_mining import config, CURRENT_DATE + + mainDir = os.path.join(config.get('Project', 'shared_folder'), "MarketPlaces/" + getMKTName() + "/HTML_Pages") fileName = getNameFromURL(url) if isDescriptionLink(url): - fullPath = r'..\CypherMarketplace\HTML_Pages\\' + CURRENT_DATE + r'\\Description\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Description\\' + fileName + '.html') else: - fullPath = r'..\CypherMarketplace\HTML_Pages\\' + CURRENT_DATE + r'\\Listing\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Listing\\' + fileName + '.html') return fullPath diff --git a/MarketPlaces/DB_Connection/db_connection.py b/MarketPlaces/DB_Connection/db_connection.py index 97296e3..a1b27ff 100644 --- a/MarketPlaces/DB_Connection/db_connection.py +++ b/MarketPlaces/DB_Connection/db_connection.py @@ -9,8 +9,8 @@ def connectDataBase(): try: - config = configparser.ConfigParser() - config.read('../../setup.ini') + from MarketPlaces.Initialization.markets_mining import config + ip = config.get('PostgreSQL', 'ip') username = config.get('PostgreSQL', 'username') password = config.get('PostgreSQL', 'password') diff --git a/MarketPlaces/DarkFox/crawler_selenium.py b/MarketPlaces/DarkFox/crawler_selenium.py index 1539c3f..3967567 100644 --- a/MarketPlaces/DarkFox/crawler_selenium.py +++ b/MarketPlaces/DarkFox/crawler_selenium.py @@ -42,16 +42,17 @@ def startCrawling(): print(driver.current_url, e) closetor(driver) - new_parse(mktName, False) + new_parse(mktName, baseURL, False) # Opens Tor Browser #prompts for ENTER input to continue def opentor(): + from MarketPlaces.Initialization.markets_mining import config + global pid print("Connecting Tor...") - path = open('../../path.txt').readline().strip() - pro = subprocess.Popen(path) + pro = subprocess.Popen(config.get('TOR', 'firefox_binary_path')) pid = pro.pid time.sleep(7.5) input('Tor Connected. Press ENTER to continue\n') @@ -93,12 +94,11 @@ def closetor(driver): # Creates FireFox 'driver' and configure its 'Profile' # to use Tor proxy and socket def createFFDriver(): - file = open('../../path.txt', 'r') - lines = file.readlines() + from MarketPlaces.Initialization.markets_mining import config - ff_binary = FirefoxBinary(lines[0].strip()) + ff_binary = FirefoxBinary(config.get('TOR', 'firefox_binary_path')) - ff_prof = FirefoxProfile(lines[1].strip()) + ff_prof = FirefoxProfile(config.get('TOR', 'firefox_profile_path')) # ff_prof.set_preference("places.history.enabled", False) # ff_prof.set_preference("privacy.clearOnShutdown.offlineApps", True) # ff_prof.set_preference("privacy.clearOnShutdown.passwords", True) @@ -120,10 +120,11 @@ def createFFDriver(): ff_prof.set_preference("javascript.enabled", False) ff_prof.update_preferences() - service = Service(lines[2].strip()) + service = Service(config.get('TOR', 'geckodriver_path')) driver = webdriver.Firefox(firefox_binary=ff_binary, firefox_profile=ff_prof, service=service) + return driver @@ -185,15 +186,14 @@ def savePage(page, url): # Gets the full path of the page to be saved along with its appropriate file name #@param: raw url as crawler crawls through every site def getFullPathName(url): + from MarketPlaces.Initialization.markets_mining import config, CURRENT_DATE + + mainDir = os.path.join(config.get('Project', 'shared_folder'), "MarketPlaces/" + getMKTName() + "/HTML_Pages") fileName = getNameFromURL(url) if isDescriptionLink(url): - fullPath = r'..\DarkFox\HTML_Pages\\' + str( - "%02d" % date.today().month) + str("%02d" % date.today().day) + str( - "%04d" % date.today().year) + r'\\' + r'Description\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Description\\' + fileName + '.html') else: - fullPath = r'..\DarkFox\HTML_Pages\\' + str( - "%02d" % date.today().month) + str("%02d" % date.today().day) + str( - "%04d" % date.today().year) + r'\\' + r'Listing\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Listing\\' + fileName + '.html') return fullPath diff --git a/MarketPlaces/DarkMatter/crawler_selenium.py b/MarketPlaces/DarkMatter/crawler_selenium.py index 564e1ea..21f6035 100644 --- a/MarketPlaces/DarkMatter/crawler_selenium.py +++ b/MarketPlaces/DarkMatter/crawler_selenium.py @@ -2,7 +2,8 @@ __author__ = 'Helium' ''' DarkMatter Marketplace Crawler (Selenium) -this is a small marketplace so next page links are not coded in +website has connection issues +not working still trying to debug ''' from selenium import webdriver @@ -25,8 +26,6 @@ from MarketPlaces.Initialization.prepare_parser import new_parse from MarketPlaces.DarkMatter.parser import darkmatter_links_parser from MarketPlaces.Utilities.utilities import cleanHTML -config = configparser.ConfigParser() -config.read('../../setup.ini') counter = 1 baseURL = 'http://darkmat3kdxestusl437urshpsravq7oqb7t3m36u2l62vnmmldzdmid.onion/' @@ -52,6 +51,8 @@ def startCrawling(): # Opens Tor Browser #prompts for ENTER input to continue def opentor(): + from MarketPlaces.Initialization.markets_mining import config + global pid print("Connecting Tor...") pro = subprocess.Popen(config.get('TOR', 'firefox_binary_path')) @@ -90,8 +91,11 @@ def closetor(driver): # Creates FireFox 'driver' and configure its 'Profile' # to use Tor proxy and socket def createFFDriver(): + from MarketPlaces.Initialization.markets_mining import config + ff_binary = FirefoxBinary(config.get('TOR', 'firefox_binary_path')) + ff_prof = FirefoxProfile(config.get('TOR', 'firefox_profile_path')) ff_prof.set_preference("places.history.enabled", False) ff_prof.set_preference("privacy.clearOnShutdown.offlineApps", True) @@ -153,12 +157,14 @@ def savePage(page, url): # Gets the full path of the page to be saved along with its appropriate file name #@param: raw url as crawler crawls through every site def getFullPathName(url): - from MarketPlaces.Initialization.markets_mining import CURRENT_DATE + from MarketPlaces.Initialization.markets_mining import config, CURRENT_DATE + + mainDir = os.path.join(config.get('Project', 'shared_folder'), "MarketPlaces/" + getMKTName() + "/HTML_Pages") fileName = getNameFromURL(url) if isDescriptionLink(url): - fullPath = r'..\DarkMatter\HTML_Pages\\' + CURRENT_DATE + r'\\Description\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Description\\' + fileName + '.html') else: - fullPath = r'..\DarkMatter\HTML_Pages\\' + CURRENT_DATE + r'\\Listing\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Listing\\' + fileName + '.html') return fullPath @@ -180,14 +186,12 @@ def getNameFromURL(url): def getInterestedLinks(): links = [] - # fraud software - links.append('http://darkmat3kdxestusl437urshpsravq7oqb7t3m36u2l62vnmmldzdmid.onion/market/products/?category=76') - # other - links.append('http://darkmat3kdxestusl437urshpsravq7oqb7t3m36u2l62vnmmldzdmid.onion/market/products/?category=133') - # # hacking + # digital + links.append('http://darkmat3kdxestusl437urshpsravq7oqb7t3m36u2l62vnmmldzdmid.onion/market/products/?category=73') + # # hack guides # links.append('http://darkmat3kdxestusl437urshpsravq7oqb7t3m36u2l62vnmmldzdmid.onion/market/products/?category=94') - # # carding - # links.append('http://darkmat3kdxestusl437urshpsravq7oqb7t3m36u2l62vnmmldzdmid.onion/market/products/?category=118') + # # services + # links.append('http://darkmat3kdxestusl437urshpsravq7oqb7t3m36u2l62vnmmldzdmid.onion/market/products/?category=117') # # software/malware # links.append('http://darkmat3kdxestusl437urshpsravq7oqb7t3m36u2l62vnmmldzdmid.onion/market/products/?category=121') @@ -220,6 +224,7 @@ def crawlForum(driver): has_next_page = True while has_next_page: list = productPages(html) + for item in list: itemURL = urlparse.urljoin(baseURL, str(item)) try: @@ -238,9 +243,9 @@ def crawlForum(driver): break try: - temp = driver.find_element(by=By.XPATH, value= - '/html/body/table[1]/tbody/tr/td/form/div/div[2]/table[2]/tbody/tr') - link = temp.find_element(by=By.CLASS_NAME, value='button page-num').get_attribute('href') + nav = driver.find_element(by=By.XPATH, value='/html/body/table[1]/tbody/tr/td/form/div/div[2]/table[2]') + a = nav.find_element(by=By.LINK_TEXT, value=">") + link = a.get_attribute('href') if link == "": raise NoSuchElementException try: @@ -268,7 +273,7 @@ def crawlForum(driver): #@param: url of any url crawled #return: true if is a description page, false if not def isDescriptionLink(url): - if 'product/' in url and '/products/?category' not in url: + if 'products/' in url and '/products/?category' not in url: return True return False diff --git a/MarketPlaces/DarkMatter/parser.py b/MarketPlaces/DarkMatter/parser.py index 54f85d7..8f71ded 100644 --- a/MarketPlaces/DarkMatter/parser.py +++ b/MarketPlaces/DarkMatter/parser.py @@ -281,10 +281,10 @@ def darkmatter_links_parser(soup): # Returning all links that should be visited by the Crawler href = [] - listing = soup.findAll('td', {"class": "lefted"}) + listing = soup.findAll('td', {"class": "lefted", 'colspan': '2'}) for a in listing: - bae = a.find('a', {"class": "lg bold"}, href=True) + bae = a.find('a', href=True) link = bae['href'] href.append(link) diff --git a/MarketPlaces/DarkTor/crawler_selenium.py b/MarketPlaces/DarkTor/crawler_selenium.py index 98beb9b..c4cbfd4 100644 --- a/MarketPlaces/DarkTor/crawler_selenium.py +++ b/MarketPlaces/DarkTor/crawler_selenium.py @@ -24,8 +24,6 @@ from MarketPlaces.Initialization.prepare_parser import new_parse from MarketPlaces.DarkTor.parser import darktor_links_parser from MarketPlaces.Utilities.utilities import cleanHTML -config = configparser.ConfigParser() -config.read('../../setup.ini') counter = 1 baseURL = 'http://zuauw53dukqdmll5p3fld26ns2gepcyfmbofobjczdni6ecmkoitnfid.onion/' @@ -51,6 +49,8 @@ def startCrawling(): # Opens Tor Browser #prompts for ENTER input to continue def opentor(): + from MarketPlaces.Initialization.markets_mining import config + global pid print("Connecting Tor...") pro = subprocess.Popen(config.get('TOR', 'firefox_binary_path')) @@ -89,6 +89,8 @@ def closetor(driver): # Creates FireFox 'driver' and configure its 'Profile' # to use Tor proxy and socket def createFFDriver(): + from MarketPlaces.Initialization.markets_mining import config + ff_binary = FirefoxBinary(config.get('TOR', 'firefox_binary_path')) ff_prof = FirefoxProfile(config.get('TOR', 'firefox_profile_path')) @@ -153,12 +155,14 @@ def savePage(page, url): # Gets the full path of the page to be saved along with its appropriate file name #@param: raw url as crawler crawls through every site def getFullPathName(url): - from MarketPlaces.Initialization.markets_mining import CURRENT_DATE + from MarketPlaces.Initialization.markets_mining import config, CURRENT_DATE + + mainDir = os.path.join(config.get('Project', 'shared_folder'), "MarketPlaces/" + getMKTName() + "/HTML_Pages") fileName = getNameFromURL(url) if isDescriptionLink(url): - fullPath = r'..\DarkTor\HTML_Pages\\' + CURRENT_DATE + r'\\Description\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Description\\' + fileName + '.html') else: - fullPath = r'..\DarkTor\HTML_Pages\\' + CURRENT_DATE + r'\\Listing\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Listing\\' + fileName + '.html') return fullPath diff --git a/MarketPlaces/DigitalThriftShop/crawler_selenium.py b/MarketPlaces/DigitalThriftShop/crawler_selenium.py index 2da39ac..e602a4f 100644 --- a/MarketPlaces/DigitalThriftShop/crawler_selenium.py +++ b/MarketPlaces/DigitalThriftShop/crawler_selenium.py @@ -24,8 +24,6 @@ from MarketPlaces.Initialization.prepare_parser import new_parse from MarketPlaces.DigitalThriftShop.parser import digitalthriftshop_links_parser from MarketPlaces.Utilities.utilities import cleanHTML -config = configparser.ConfigParser() -config.read('../../setup.ini') counter = 1 baseURL = 'http://kw4zlnfhxje7top26u57iosg55i7dzuljjcyswo2clgc3mdliviswwyd.onion/' @@ -89,6 +87,9 @@ def closetor(driver): # Creates FireFox 'driver' and configure its 'Profile' # to use Tor proxy and socket def createFFDriver(): + from MarketPlaces.Initialization.markets_mining import config + + ff_binary = FirefoxBinary(config.get('TOR', 'firefox_binary_path')) ff_prof = FirefoxProfile(config.get('TOR', 'firefox_profile_path')) @@ -153,12 +154,14 @@ def savePage(page, url): # Gets the full path of the page to be saved along with its appropriate file name #@param: raw url as crawler crawls through every site def getFullPathName(url): - from MarketPlaces.Initialization.markets_mining import CURRENT_DATE + from MarketPlaces.Initialization.markets_mining import config, CURRENT_DATE + + mainDir = os.path.join(config.get('Project', 'shared_folder'), "MarketPlaces/" + getMKTName() + "/HTML_Pages") fileName = getNameFromURL(url) if isDescriptionLink(url): - fullPath = r'..\DigitalThriftShop\HTML_Pages\\' + CURRENT_DATE + r'\\Description\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Description\\' + fileName + '.html') else: - fullPath = r'..\DigitalThriftShop\HTML_Pages\\' + CURRENT_DATE + r'\\Listing\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Listing\\' + fileName + '.html') return fullPath diff --git a/MarketPlaces/Initialization/geckodriver.log b/MarketPlaces/Initialization/geckodriver.log index 0b4b518..aaee97e 100644 --- a/MarketPlaces/Initialization/geckodriver.log +++ b/MarketPlaces/Initialization/geckodriver.log @@ -10187,7 +10187,117 @@ console.error: ({}) JavaScript error: resource://devtools/shared/DevToolsUtils.js, line 670: Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/f509c187-d615-43a3-a180-58ce13333c99' JavaScript error: resource://devtools/shared/DevToolsUtils.js, line 670: Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/60a94659-6501-4485-9534-50503b79d5a4' JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofileK69fD1\thumbnails) because it does not exist --debugging-port" "52804" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofileLacTlO" +console.warn: "Resource of root-node was not found." +console.warn: "Resource of root-node was not found." +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.warn: "Resource of root-node was not found." +console.warn: "Resource of root-node was not found." +console.warn: "Resource of root-node was not found." +console.warn: "Resource of root-node was not found." +JavaScript error: resource://devtools/server/actors/highlighters/utils/markup.js, line 141: TypeError: window.document.documentElement is null +console.warn: "Resource of root-node was not found." +console.error: (new Error("Protocol error (noSuchActor): No such actor for ID: server0.conn0.child4/domnode899 from: server0.conn0.child4/domnode899", "resource://devtools/shared/protocol/Front.js", 362)) +console.log: "TorConnect: beginBootstrap()" +console.log: "TorConnect: Try transitioning from Initial to Bootstrapping" +console.log: "TorConnect: Exited Initial state" +JavaScript error: resource:///modules/TorConnectParent.jsm, line 110: InvalidStateError: JSWindowActorParent.sendAsyncMessage: JSWindowActorParent cannot send at the moment +console.log: "TorConnect: Entering Bootstrapping state" +1687900659804 Marionette INFO Stopped listening on port 52750 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first. +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofileK69fD1\thumbnails) because it does not exist +[Parent 6092, IPC I/O Parent] WARNING: file /var/tmp/build/firefox-b6010b1466c9/ipc/chromium/src/base/process_util_win.cc:167 + +###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + +1687900660002 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +/ipc/chromium/src/base/process_util_win.cc:167 + +###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + +1687899446282 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1687899481019 geckodriver INFO Listening on 127.0.0.1:52853 +1687899485284 mozrunner::runner INFO Running command: "C:\\Users\\\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "52854" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofilekeL3Un" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: init()" +console.log: "TorConnect: Entering Initial state" +console.log: "TorConnect: Observed profile-after-change" +console.log: "TorConnect: Observing topic 'TorProcessExited'" +console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: Observing topic 'torsettings:ready'" +console.log: "TorSettings: Observed profile-after-change" +1687899485991 Marionette INFO Marionette enabled +console.log: "TorConnect: Will load after bootstrap => [about:blank]" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:52854/devtools/browser/1a206e40-2906-444c-8941-bf802d89326f +1687899488352 Marionette INFO Listening on port 52859 +1687899488952 RemoteAgent WARN TLS certificate errors will be ignored for this session +<<<<<<< HEAD +>>>>>>> e327ce67264bf947eb8fe1873f1aeffac0b830d9 +======= +SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/635360d9-c669-4d56-9967-2f26122c3313'" nsresult: "0x805303f4 ()" location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670" data: yes] +Stack: mainThreadFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15 +mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10 +_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22 +urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17 +_resurrectSource@resource://devtools/server/actors/thread.js:2142:35 +addAllSources@resource://devtools/server/actors/thread.js:1509:14 +watch@resource://devtools/server/actors/resources/sources.js:52:17 +watchResources@resource://devtools/server/actors/resources/index.js:239:19 +_watchTargetResources@resource://devtools/server/actors/targets/target-actor-mixin.js:156:24 +addWatcherDataEntry@resource://devtools/server/actors/targets/target-actor-mixin.js:47:20 +_addWatcherDataEntry@resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm:483:24 +receiveMessage@resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm:425:21 +Line: 670, column: 0 +console.error: ({}) +SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/a9b866dd-f313-4dea-ba11-6336b9be6813'" nsresult: "0x805303f4 ()" location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670" data: yes] +Stack: mainThreadFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15 +mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10 +_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22 +urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17 +_resurrectSource@resource://devtools/server/actors/thread.js:2142:35 +addAllSources@resource://devtools/server/actors/thread.js:1509:14 +watch@resource://devtools/server/actors/resources/sources.js:52:17 +watchResources@resource://devtools/server/actors/resources/index.js:239:19 +_watchTargetResources@resource://devtools/server/actors/targets/target-actor-mixin.js:156:24 +addWatcherDataEntry@resource://devtools/server/actors/targets/target-actor-mixin.js:47:20 +_addWatcherDataEntry@resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm:483:24 +receiveMessage@resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm:425:21 +Line: 670, column: 0 +console.error: ({}) +JavaScript error: resource://devtools/shared/DevToolsUtils.js, line 670: Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/635360d9-c669-4d56-9967-2f26122c3313' +JavaScript error: resource://devtools/shared/DevToolsUtils.js, line 670: Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/a9b866dd-f313-4dea-ba11-6336b9be6813' +1687900055842 Marionette INFO Stopped listening on port 52859 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first. +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofilekeL3Un\thumbnails) because it does not exist +[Parent 8848, IPC I/O Parent] WARNING: file /var/tmp/build/firefox-b6010b1466c9/ipc/chromium/src/base/process_util_win.cc:167 +1687900056058 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1687900077721 geckodriver INFO Listening on 127.0.0.1:52920 +1687900081472 mozrunner::runner INFO Running command: "C:\\Users\\\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "52921" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofileCmJBBa" console.log: "TorSettings: loadFromPrefs()" console.log: "TorConnect: init()" console.log: "TorConnect: Entering Initial state" @@ -10196,7 +10306,7 @@ console.log: "TorConnect: Observing topic 'TorProcessExited'" console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'" console.log: "TorConnect: Observing topic 'torsettings:ready'" console.log: "TorSettings: Observed profile-after-change" -1687899141198 Marionette INFO Marionette enabled +1687900082076 Marionette INFO Marionette enabled console.log: "TorConnect: Will load after bootstrap => [about:blank]" console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. @@ -10204,28 +10314,78 @@ JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't fin JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined -DevTools listening on ws://localhost:52804/devtools/browser/2c8f592d-fa82-4210-8065-6502e2bef399 -1687899143622 Marionette INFO Listening on port 52809 -1687899143926 RemoteAgent WARN TLS certificate errors will be ignored for this session -JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofileLacTlO\thumbnails) because it does not exist -1687899446123 Marionette INFO Stopped listening on port 52809 +DevTools listening on ws://localhost:52921/devtools/browser/faadb1be-5099-404d-8141-a79c255e9fd4 +1687900083816 Marionette INFO Listening on port 52926 +1687900084097 RemoteAgent WARN TLS certificate errors will be ignored for this session +SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/d3c4a63a-a8f2-4fa3-a1aa-43c01c357db8'" nsresult: "0x805303f4 ()" location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670" data: yes] +Stack: mainThreadFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15 +mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10 +_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22 +urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17 +_resurrectSource@resource://devtools/server/actors/thread.js:2142:35 +addAllSources@resource://devtools/server/actors/thread.js:1509:14 +watch@resource://devtools/server/actors/resources/sources.js:52:17 +watchResources@resource://devtools/server/actors/resources/index.js:239:19 +_watchTargetResources@resource://devtools/server/actors/targets/target-actor-mixin.js:156:24 +addWatcherDataEntry@resource://devtools/server/actors/targets/target-actor-mixin.js:47:20 +_addWatcherDataEntry@resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm:483:24 +receiveMessage@resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm:425:21 +Line: 670, column: 0 +console.error: ({}) +SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/7bd4bd6a-e390-47d6-81bd-8026d113f159'" nsresult: "0x805303f4 ()" location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670" data: yes] +Stack: mainThreadFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15 +mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10 +_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22 +urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17 +_resurrectSource@resource://devtools/server/actors/thread.js:2142:35 +addAllSources@resource://devtools/server/actors/thread.js:1509:14 +watch@resource://devtools/server/actors/resources/sources.js:52:17 +watchResources@resource://devtools/server/actors/resources/index.js:239:19 +_watchTargetResources@resource://devtools/server/actors/targets/target-actor-mixin.js:156:24 +addWatcherDataEntry@resource://devtools/server/actors/targets/target-actor-mixin.js:47:20 +_addWatcherDataEntry@resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm:483:24 +receiveMessage@resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm:425:21 +Line: 670, column: 0 +console.error: ({}) +JavaScript error: resource://devtools/shared/DevToolsUtils.js, line 670: Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/d3c4a63a-a8f2-4fa3-a1aa-43c01c357db8' +JavaScript error: resource://devtools/shared/DevToolsUtils.js, line 670: Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/7bd4bd6a-e390-47d6-81bd-8026d113f159' +console.error: (new TypeError("container.editor is undefined", "resource://devtools/client/inspector/markup/markup.js", 1619)) +1687900441910 Marionette INFO Stopped listening on port 52926 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first. +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofileCmJBBa\thumbnails) because it does not exist + +###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + +1687900442180 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +vaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:52949/devtools/browser/782cf46e-4100-402f-8788-f4919df8e22d +1687900412838 Marionette INFO Listening on port 52954 +1687900413352 RemoteAgent WARN TLS certificate errors will be ignored for this session +1687900439655 Marionette INFO Stopped listening on port 52954 JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData] !!! error running onStopped callback: TypeError: callback is not a function JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first. -JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofileLacTlO\thumbnails) because it does not exist -[Parent 7464, IPC I/O Parent] WARNING: file /var/tmp/build/firefox-b6010b1466c9/ipc/chromium/src/base/process_util_win.cc:167 +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofileyuuH2U\thumbnails) because it does not exist ###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost -1687899446282 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +1687900440059 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 unwatchForTabs()@TargetList.jsm:70 unwatchForTargets()@TargetList.jsm:37 destructor()@TargetList.jsm:109 stop()@CDP.jsm:104 close()@RemoteAgent.jsm:138 -1687899481019 geckodriver INFO Listening on 127.0.0.1:52853 -1687899485284 mozrunner::runner INFO Running command: "C:\\Users\\\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "52854" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofilekeL3Un" +1687900526521 geckodriver INFO Listening on 127.0.0.1:52998 +1687900531668 mozrunner::runner INFO Running command: "C:\\Users\\\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "52999" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofiletprjJy" console.log: "TorSettings: loadFromPrefs()" console.log: "TorConnect: init()" console.log: "TorConnect: Entering Initial state" @@ -10234,7 +10394,7 @@ console.log: "TorConnect: Observing topic 'TorProcessExited'" console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'" console.log: "TorConnect: Observing topic 'torsettings:ready'" console.log: "TorSettings: Observed profile-after-change" -1687899485991 Marionette INFO Marionette enabled +1687900532518 Marionette INFO Marionette enabled console.log: "TorConnect: Will load after bootstrap => [about:blank]" console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. @@ -10242,7 +10402,896 @@ JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't fin JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined -DevTools listening on ws://localhost:52854/devtools/browser/1a206e40-2906-444c-8941-bf802d89326f -1687899488352 Marionette INFO Listening on port 52859 -1687899488952 RemoteAgent WARN TLS certificate errors will be ignored for this session ->>>>>>> e327ce67264bf947eb8fe1873f1aeffac0b830d9 +DevTools listening on ws://localhost:52999/devtools/browser/b89e31a5-754c-4464-b48e-e02f55b2e6fd +1687900535097 Marionette INFO Listening on port 53004 +1687900535396 RemoteAgent WARN TLS certificate errors will be ignored for this session +1687900608906 Marionette INFO Stopped listening on port 53004 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first. +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofiletprjJy\thumbnails) because it does not exist +1687900609382 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1687900697513 geckodriver INFO Listening on 127.0.0.1:53060 +1687900700197 mozrunner::runner INFO Running command: "C:\\Users\\\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "53061" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofileAJcrYO" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: init()" +console.log: "TorConnect: Entering Initial state" +console.log: "TorConnect: Observed profile-after-change" +console.log: "TorConnect: Observing topic 'TorProcessExited'" +console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: Observing topic 'torsettings:ready'" +console.log: "TorSettings: Observed profile-after-change" +1687900700626 Marionette INFO Marionette enabled +console.log: "TorConnect: Will load after bootstrap => [about:blank]" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:53061/devtools/browser/955e7c03-cc69-42a0-b58d-03def93f9c9d +1687900701960 Marionette INFO Listening on port 53066 +1687900702082 RemoteAgent WARN TLS certificate errors will be ignored for this session +SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/40cf853b-9de5-4bee-8aea-aa5059ae8b6d'" nsresult: "0x805303f4 ()" location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670" data: yes] +Stack: mainThreadFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15 +mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10 +_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22 +urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17 +_resurrectSource@resource://devtools/server/actors/thread.js:2142:35 +addAllSources@resource://devtools/server/actors/thread.js:1509:14 +watch@resource://devtools/server/actors/resources/sources.js:52:17 +watchResources@resource://devtools/server/actors/resources/index.js:239:19 +_watchTargetResources@resource://devtools/server/actors/targets/target-actor-mixin.js:156:24 +addWatcherDataEntry@resource://devtools/server/actors/targets/target-actor-mixin.js:47:20 +_addWatcherDataEntry@resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm:483:24 +receiveMessage@resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm:425:21 +Line: 670, column: 0 +console.error: ({}) +SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/76d21179-e63d-4686-bef9-0fa8c2e2fa8a'" nsresult: "0x805303f4 ()" location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670" data: yes] +Stack: mainThreadFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15 +mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10 +_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22 +urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17 +_resurrectSource@resource://devtools/server/actors/thread.js:2142:35 +addAllSources@resource://devtools/server/actors/thread.js:1509:14 +watch@resource://devtools/server/actors/resources/sources.js:52:17 +watchResources@resource://devtools/server/actors/resources/index.js:239:19 +_watchTargetResources@resource://devtools/server/actors/targets/target-actor-mixin.js:156:24 +addWatcherDataEntry@resource://devtools/server/actors/targets/target-actor-mixin.js:47:20 +_addWatcherDataEntry@resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm:483:24 +receiveMessage@resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm:425:21 +Line: 670, column: 0 +console.error: ({}) +JavaScript error: resource://devtools/shared/DevToolsUtils.js, line 670: Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/40cf853b-9de5-4bee-8aea-aa5059ae8b6d' +JavaScript error: resource://devtools/shared/DevToolsUtils.js, line 670: Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/76d21179-e63d-4686-bef9-0fa8c2e2fa8a' +JavaScript error: resource://gre/actors/AutoCompleteChild.jsm, line 125: Error: Invalid autocomplete selectedIndex +console.warn: "Resource of root-node was not found." +console.warn: "Resource of root-node was not found." +SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/40cf853b-9de5-4bee-8aea-aa5059ae8b6d'" nsresult: "0x805303f4 ()" location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670" data: yes] +Stack: mainThreadFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15 +mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10 +_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22 +urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17 +_resurrectSource@resource://devtools/server/actors/thread.js:2142:35 +addAllSources@resource://devtools/server/actors/thread.js:1509:14 +_onWindowReady@resource://devtools/server/actors/thread.js:1818:12 +_emit@resource://devtools/shared/event-emitter.js:226:34 +emit@resource://devtools/shared/event-emitter.js:172:18 +emit@resource://devtools/shared/event-emitter.js:324:18 +_windowReady@resource://devtools/server/actors/targets/browsing-context.js:1406:10 +DebuggerProgressListener.prototype.onWindowCreated<@resource://devtools/server/actors/targets/browsing-context.js:1761:23 +exports.makeInfallible/<@resource://devtools/shared/ThreadSafeDevToolsUtils.js:103:22 +Line: 670, column: 0 +console.error: ({}) +SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/76d21179-e63d-4686-bef9-0fa8c2e2fa8a'" nsresult: "0x805303f4 ()" location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670" data: yes] +Stack: mainThreadFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15 +mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10 +_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22 +urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17 +_resurrectSource@resource://devtools/server/actors/thread.js:2142:35 +addAllSources@resource://devtools/server/actors/thread.js:1509:14 +_onWindowReady@resource://devtools/server/actors/thread.js:1818:12 +_emit@resource://devtools/shared/event-emitter.js:226:34 +emit@resource://devtools/shared/event-emitter.js:172:18 +emit@resource://devtools/shared/event-emitter.js:324:18 +_windowReady@resource://devtools/server/actors/targets/browsing-context.js:1406:10 +DebuggerProgressListener.prototype.onWindowCreated<@resource://devtools/server/actors/targets/browsing-context.js:1761:23 +exports.makeInfallible/<@resource://devtools/shared/ThreadSafeDevToolsUtils.js:103:22 +Line: 670, column: 0 +console.error: ({}) +JavaScript error: resource://devtools/shared/DevToolsUtils.js, line 670: Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/40cf853b-9de5-4bee-8aea-aa5059ae8b6d' +JavaScript error: resource://devtools/shared/DevToolsUtils.js, line 670: Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/76d21179-e63d-4686-bef9-0fa8c2e2fa8a' +console.warn: "Resource of root-node was not found." +SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/b25e7d24-9dbe-4a0d-8967-f4393fe808e5'" nsresult: "0x805303f4 ()" location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670" data: yes] +Stack: mainThreadFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15 +mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10 +_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22 +urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17 +_resurrectSource@resource://devtools/server/actors/thread.js:2142:35 +addAllSources@resource://devtools/server/actors/thread.js:1509:14 +_onWindowReady@resource://devtools/server/actors/thread.js:1818:12 +_emit@resource://devtools/shared/event-emitter.js:226:34 +emit@resource://devtools/shared/event-emitter.js:172:18 +emit@resource://devtools/shared/event-emitter.js:324:18 +_windowReady@resource://devtools/server/actors/targets/browsing-context.js:1406:10 +DebuggerProgressListener.prototype.onWindowCreated<@resource://devtools/server/actors/targets/browsing-context.js:1761:23 +exports.makeInfallible/<@resource://devtools/shared/ThreadSafeDevToolsUtils.js:103:22 +Line: 670, column: 0 +console.error: ({}) +SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/0645a919-7a1c-4765-a229-b7cd76aeecb8'" nsresult: "0x805303f4 ()" location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670" data: yes] +Stack: mainThreadFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15 +mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10 +_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22 +urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17 +_resurrectSource@resource://devtools/server/actors/thread.js:2142:35 +addAllSources@resource://devtools/server/actors/thread.js:1509:14 +_onWindowReady@resource://devtools/server/actors/thread.js:1818:12 +_emit@resource://devtools/shared/event-emitter.js:226:34 +emit@resource://dev1687902281118 Marionette INFO Stopped listening on port 53225 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first. +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofilePVaQxK\thumbnails) because it does not exist +1687902281390 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +dFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15 +mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10 +_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22 +urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17 +_resurrectSource@resource://devtools/server/actors/thread.js:2142:35 +addAllSources@resource://devtools/server/actors/thread.js:1509:14 +_onWindowReady@resource://devtools/server/actors/thread.js:1818:12 +_emit@resource://devtools/shared/event-emitter.js:226:34 +emit@resource://devtools/shared/event-emitter.js:172:18 +emit@resource://devtools/shared/event-emitter.js:324:18 +_windowReady@resource://devtools/server/actors/targets/browsing-context.js:1406:10 +DebuggerProgressListener.prototype.onWindowCreated<@resource://devtools/server/actors/targets/browsing-context.js:1761:23 +exports.makeInfallible/<@resource://devtools/shared/ThreadSafeDevToolsUtils.js:103:22 +Line: 670, column: 0 +console.error: ({}) +SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/76d21179-e63d-4686-bef9-0fa8c2e2fa8a'" nsresult: "0x805303f4 ()" location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670" data: yes] +Stack: mainThreadFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15 +mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10 +_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22 +urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17 +_resurrectSource@resource://devtools/server/actors/thread.js:2142:35 +addAllSources@resource://devtools/server/actors/thread.js:1509:14 +_onWindowReady@resource://devtools/server/actors/thread.js:1818:12 +_emit@resource://devtools/shared/event-emitter.js:226:34 +emit@resource://devtools/shared/event-emitter.js:172:18 +emit@resource://devtools/shared/event-emitter.js:324:18 +_windowReady@resource://devtools/server/actors/targets/browsing-context.js:1406:10 +DebuggerProgressListener.prototype.onWindowCreated<@resource://devtools/server/actors/targets/browsing-context.js:1761:23 +exports.makeInfallible/<@resource://devtools/shared/ThreadSafeDevToolsUtils.js:103:22 +Line: 670, column: 0 +console.error: ({}) +JavaScript error: resource://devtools/shared/DevToolsUtils.js, line 670: Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/40cf853b-9de5-4bee-8aea-aa5059ae8b6d' +JavaScript error: resource://devtools/shared/DevToolsUtils.js, line 670: Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/76d21179-e63d-4686-bef9-0fa8c2e2fa8a' +console.warn: "Resource of root-node was not found." +SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/b25e7d24-9dbe-4a0d-8967-f4393fe808e5'" nsresult: "0x805303f4 ()" location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670" data: yes] +Stack: mainThreadFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15 +mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10 +_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22 +urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17 +_resurrectSource@resource://devtools/server/actors/thread.js:2142:35 +addAllSources@resource://devtools/server/actors/thread.js:1509:14 +_onWindowReady@resource://devtools/server/actors/thread.js:1818:12 +_emit@resource://devtools/shared/event-emitter.js:226:34 +emit@resource://devtools/shared/event-emitter.js:172:18 +emit@resource://devtools/shared/event-emitter.js:324:18 +_windowReady@resource://devtools/server/actors/targets/browsing-context.js:1406:10 +DebuggerProgressListener.prototype.onWindowCreated<@resource://devtools/server/actors/targets/browsing-context.js:1761:23 +exports.makeInfallible/<@resource://devtools/shared/ThreadSafeDevToolsUtils.js:103:22 +Line: 670, column: 0 +console.error: ({}) +SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/0645a919-7a1c-4765-a229-b7cd76aeecb8'" nsresult: "0x805303f4 ()" location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670" data: yes] +Stack: mainThreadFetch/<@resource://devtools/shJavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofile6O2W01\thumbnails) because it does not exist +1687903111317 Marionette INFO Stopped listening on port 53285 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first. +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofile6O2W01\thumbnails) because it does not exist +1687903111599 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/40cf853b-9de5-4bee-8aea-aa5059ae8b6d'" nsresult: "0x805303f4 ()" location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670" data: yes] +Stack: mainThreadFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15 +mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10 +_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22 +urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17 +_resurrectSource@resource://devtools/server/actors/thread.js:2142:35 +addAllSources@resource://devtools/server/actors/thread.js:1509:14 +_onWindowReady@resource://devtools/server/actors/thread.js:1818:12 +_emit@resource://devtools/shared/event-emitter.js:226:34 +emit@resource://devtools/shared/event-emitter.js:172:18 +emit@resource://devtools/shared/event-emitter.js:324:18 +_windowReady@resource://devtools/server/actors/targets/browsing-context.js:1406:10 +DebuggerProgressListener.prototype.onWindowCreated<@resource://devtools/server/actors/targets/browsing-context.js:1761:23 +exports.makeInfallible/<@resource://devtools/shared/ThreadSafeDevToolsUtils.js:103:22 +Line: 670, column: 0 +console.error: ({}) +SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/76d21179-e63d-4686-bef9-0fa8c2e2fa8a'" nsresult: "0x805303f4 ()" location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670" data: yes] +Stack: mainThreadFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15 +mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10 +_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22 +urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17 +_resurrectSource@resource://devtools/server/actors/thread.js:2142:35 +addAllSources@resource://devtools/server/actors/thread.js:1509:14 +_onWindowReady@resource://devtools/server/actors/thread.js:1818:12 +_emit@resource://devtools/shared/event-emitter.js:226:34 +emit@resource://devtools/shared/event-emitter.js:172:18 +emit@resource://devtools/shared/event-emitter.js:324:18 +_windowReady@resource://devtools/server/actors/targets/browsing-context.js:1406:10 +DebuggerProgressListener.prototype.onWindowCreated<@resource://devtools/server/actors/targets/browsing-context.js:1761:23 +exports.makeInfallible/<@resource://devtools/shared/ThreadSafeDevToolsUtils.js:103:22 +Line: 670, column: 0 +console.error: ({}) +JavaScript error: resource://devtools/shared/DevToolsUtils.js, line 670: Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/40cf853b-9de5-4bee-8aea-aa5059ae8b6d' +JavaScript error: resource://devtools/shared/DevToolsUtils.js, line 670: Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/76d21179-e63d-4686-bef9-0fa8c2e2fa8a' +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofileAJcrYO\thumbnails) because it does not exist +JavaScript error: moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/nscl/content/patchWindow.js, line 294: TypeError: can't access dead object +JavaScript error: moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/nscl/content/patchWindow.js, line 294: TypeError: can't access dead object +JavaScript error: moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/nscl/content/patchWindow.js, line 294: TypeError: can't access dead object +console.warn: "Resource of root-node was not found." +SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/b25e7d24-9dbe-4a0d-8967-f4393fe808e5'" nsresult: "0x805303f4 ()" location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670" data: yes] +Stack: mainThreadFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15 +mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10 +_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22 +urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17 +_resurrectSource@resource://devtools/server/actors/thread.js:2142:35 +addAllSources@resource://devtools/server/actors/thread.js:1509:14 +_onWindowReady@resource://devtools/server/actors/thread.js:1818:12 +_emit@resource://devtools/shared/event-emitter.js:226:34 +emit@resource://devtools/shared/event-emitter.js:172:18 +emit@resource://devtools/shared/event-emitter.js:324:18 +_windowReady@resource://devtools/server/actors/targets/browsing-context.js:1406:10 +DebuggerProgressListener.prototype.onWindowCreated<@resource://devtools/server/actors/targets/browsing-context.js:1761:23 +exports.makeInfallible/<@resource://devtools/shared/ThreadSafeDevToolsUtils.js:103:22 +Line: 670, column: 0 +console.error: ({}) +SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/0645a919-7a1c-4765-a229-b7cd76aeecb8'" nsresult: "0x805303f4 ()" location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670" data: yes] +Stack: mainThreadFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15 +mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10 +_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22 +urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17 +_resurrectSource@resource://devtools/server/actors/thread.js:2142:35 +addAllSources@resource://devtools/server/actors/thread.js:1509:14 +_onWindowReady@resource://devtools/server/actors/thread.js:1818:12 +_emit@resource://devtools/shared/event-emitter.js:226:34 +emit@resource://devtools/shared/event-emitter.js:172:18 +emit@resource://devtools/shared/event-emitter.js:324:18 +_windowReady@resource://devtools/server/actors/targets/browsing-context.js:1406:10 +DebuggerProgressListener.prototype.onWindowCreated<@resource://devtools/server/actors/targets/browsing-context.js:1761:23 +exports.makeInfallible/<@resource://devtools/shared/ThreadSafeDevToolsUtils.js:103:22 +Line: 670, column: 0 +console.error: ({}) +JavaScript error: resource://devtools/shared/DevToolsUtils.js, line 670: Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/b25e7d24-9dbe-4a0d-8967-f4393fe808e5' +JavaScript error: resource://devtools/shared/DevToolsUtils.js, line 670: Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/0645a919-7a1c-4765-a229-b7cd76aeecb8' +console.warn: "Resource of root-node was not found." +SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/40cf853b-9de5-4bee-8aea-aa5059ae8b6d'" nsresult: "0x805303f4 ()" location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670" data: yes] +Stack: mainThreadFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15 +mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10 +_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22 +urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17 +_resurrectSource@resource://devtools/server/actors/thread.js:2142:35 +addAllSources@resource://devtools/server/actors/thread.js:1509:14 +_onWindowReady@resource://devtools/server/actors/thread.js:1818:12 +_emit@resource://devtools/shared/event-emitter.js:226:34 +emit@resource://devtools/shared/event-emitter.js:172:18 +emit@resource://devtools/shared/event-emitter.js:324:18 +_windowReady@resource://devtools/server/actors/targets/browsing-context.js:1406:10 +DebuggerProgressListener.prototype.onWindowCreated<@resource://devtools/server/actors/targets/browsing-context.js:1761:23 +exports.makeInfallible/<@resource://devtools/shared/ThreadSafeDevToolsUtils.js:103:22 +Line: 670, column: 0 +console.error: ({}) +SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/76d21179-e63d-4686-bef9-0fa8c2e2fa8a'" nsresult: "0x805303f4 ()" location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670" data: yes] +Stack: mainThreadFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15 +mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10 +_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22 +urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17 +_resurrectSource@resource://devtools/server/actors/thread.js:2142:35 +addAllSources@resource://devtools/server/actors/thread.js:1509:14 +_onWindowReady@resource://devtools/server/actors/thread.js:1818:12 +_emit@resource://devtools/shared/event-emitter.js:226:34 +emit@resource://devtools/shared/event-emitter.js:172:18 +emit@resource://devtools/shared/event-emitter.js:324:18 +_windowReady@resource://devtools/server/actors/targets/browsing-context.js:1406:10 +DebuggerProgressListener.prototype.onWindowCreated<@resource://devtools/server/actors/targets/browsing-context.js:1761:23 +exports.makeInfallible/<@resource://devtools/shared/ThreadSafeDevToolsUtils.js:103:22 +Line: 670, column: 0 +console.error: ({}) +JavaScript error: resource://devtools/shared/DevToolsUtils.js, line 670: Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/40cf853b-9de5-4bee-8aea-aa5059ae8b6d' +JavaScript error: resource://devtools/shared/DevToolsUtils.js, line 670: Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/76d21179-e63d-4686-bef9-0fa8c2e2fa8a' +console.warn: "Resource of root-node was not found." +console.warn: "Resource of root-node was not found." +SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/40cf853b-9de5-4bee-8aea-aa5059ae8b6d'" nsresult: "0x805303f4 ()" location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670" data: yes] +Stack: mainThreadFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15 +mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10 +_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22 +urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17 +_resurrectSource@resource://devtools/server/actors/thread.js:2142:35 +addAllSources@resource://devtools/server/actors/thread.js:1509:14 +_onWindowReady@resource://devtools/server/actors/thread.js:1818:12 +_emit@resource://devtools/shared/event-emitter.js:226:34 +emit@resource://devtools/shared/event-emitter.js:172:18 +emit@resource://devtools/shared/event-emitter.js:324:18 +_windowReady@resource://devtools/server/actors/targets/browsing-context.js:1406:10 +DebuggerProgressListener.prototype.onWindowCreated<@resource://devtools/server/actors/targets/browsing-context.js:1761:23 +exports.makeInfallible/<@resource://devtools/shared/ThreadSafeDevToolsUtils.js:103:22 +Line: 670, column: 0 +console.error: ({}) +SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/76d21179-e63d-4686-bef9-0fa8c2e2fa8a'" nsresult: "0x805303f4 ()" location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670" data: yes] +Stack: mainThreadFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15 +mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10 +_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22 +urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17 +_resurrectSource@resource://devtools/server/actors/thread.js:2142:35 +addAllSources@resource://devtools/server/actors/thread.js:1509:14 +_onWindowReady@resource://devtools/server/actors/thread.js:1818:12 +_emit@resource://devtools/shared/event-emitter.js:226:34 +emit@resource://devtools/shared/event-emitter.js:172:18 +emit@resource://devtools/shared/event-emitter.js:324:18 +_windowReady@resource://devtools/server/actors/targets/browsing-context.js:1406:10 +DebuggerProgressListener.prototype.onWindowCreated<@resource://devtools/server/actors/targets/browsing-context.js:1761:23 +exports.makeInfallible/<@resource://devtools/shared/ThreadSafeDevToolsUtils.js:103:22 +Line: 670, column: 0 +console.error: ({}) +JavaScript error: resource://devtools/shared/DevToolsUtils.js, line 670: Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/40cf853b-9de5-4bee-8aea-aa5059ae8b6d' +JavaScript error: resource://devtools/shared/DevToolsUtils.js, line 670: Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/76d21179-e63d-4686-bef9-0fa8c2e2fa8a' +console.warn: "Resource of root-node was not found." +console.warn: "Resource of root-node was not found." +SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/df5402e9-1dc6-4871-8daa-d4682021b53f'" nsresult: "0x805303f4 ()" location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670" data: yes] +Stack: mainThreadFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15 +mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10 +_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22 +urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17 +_resurrectSource@resource://devtools/server/actors/thread.js:2142:35 +addAllSources@resource://devtools/server/actors/thread.js:1509:14 +_onWindowReady@resource://devtools/server/actors/thread.js:1818:12 +_emit@resource://devtools/shared/event-emitter.js:226:34 +emit@resource://devtools/shared/event-emitter.js:172:18 +emit@resource://devtools/shared/event-emitter.js:324:18 +_windowReady@resource://devtools/server/actors/targets/browsing-context.js:1406:10 +DebuggerProgressListener.prototype.onWindowCreated<@resource://devtools/server/actors/targets/browsing-context.js:1761:23 +exports.makeInfallible/<@resource://devtools/shared/ThreadSafeDevToolsUtils.js:103:22 +Line: 670, column: 0 +console.error: ({}) +SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/b313acd2-a254-4b83-a49a-d5f065af7264'" nsresult: "0x805303f4 ()" location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670" data: yes] +Stack: mainThreadFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15 +mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10 +_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22 +urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17 +_resurrectSource@resource://devtools/server/actors/thread.js:2142:35 +addAllSources@resource://devtools/server/actors/thread.js:1509:14 +_onWindowReady@resource://devtools/server/actors/thread.js:1818:12 +_emit@resource://devtools/shared/event-emitter.js:226:34 +emit@resource://devtools/shared/event-emitter.js:172:18 +emit@resource://devtools/shared/event-emitter.js:324:18 +_windowReady@resource://devtools/server/actors/targets/browsing-context.js:1406:10 +DebuggerProgressListener.prototype.onWindowCreated<@resource://devtools/server/actors/targets/browsing-context.js:1761:23 +exports.makeInfallible/<@resource://devtools/shared/ThreadSafeDevToolsUtils.js:103:22 +Line: 670, column: 0 +console.error: ({}) +JavaScript error: resource://devtools/shared/DevToolsUtils.js, line 670: Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/df5402e9-1dc6-4871-8daa-d4682021b53f' +JavaScript error: resource://devtools/shared/DevToolsUtils.js, line 670: Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/b313acd2-a254-4b83-a49a-d5f065af7264' +console.warn: "Resource of root-node was not found." +console.warn: "Resource of root-node was not found." +3394 +1687903160088 mozrunner::runner INFO Running command: "C:\\Users\\\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "53395" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofile82DJLY" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: init()" +console.log: "TorConnect: Entering Initial state" +console.log: "TorConnect: Observed profile-after-change" +console.log: "TorConnect: Observing topic 'TorProcessExited'" +console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: Observing topic 'torsettings:ready'" +console.log: "TorSettings: Observed profile-after-change" +1687903160748 Marionette INFO Marionette enabled +console.log: "TorConnect: Will load after bootstrap => [about:blank]" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:53395/devtools/browser/8603e572-fad4-4fa1-8430-102c9347d567 +1687903162751 Marionette INFO Listening on port 53400 +1687903162869 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofile82DJLY\thumbnails) because it does not exist +1687903290748 Marionette INFO Stopped listening on port 53400 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData] +!!! error running onStopped callback: TypeError: callback is not a function +JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first. +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofile82DJLY\thumbnails) because it does not exist + +###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + +1687903290986 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1687903332011 geckodriver INFO Listening on 127.0.0.1:53446 +1687903335536 mozrunner::runner INFO Running command: "C:\\Users\\\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "53447" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofilebmJgR0" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: init()" +console.log: "TorConnect: Entering Initial state" +console.log: "TorConnect: Observed profile-after-change" +console.log: "TorConnect: Observing topic 'TorProcessExited'" +console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: Observing topic 'torsettings:ready'" +console.log: "TorSettings: Observed profile-after-change" +1687903336065 Marionette INFO Marionette enabled +console.log: "TorConnect: Will load after bootstrap => [about:blank]" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:53447/devtools/browser/29b13ae1-3bf7-4653-9b3c-eccbfb573e7e +1687903337842 Marionette INFO Listening on port 53452 +1687903338302 RemoteAgent WARN TLS certificate errors will be ignored for this session +1687903399417 Marionette INFO Stopped listening on port 53452 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first. +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofilebmJgR0\thumbnails) because it does not exist +1687903399939 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1687903427088 geckodriver INFO Listening on 127.0.0.1:53498 +1687903430973 mozrunner::runner INFO Running command: "C:\\Users\\\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "53499" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofileb5l8aN" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: init()" +console.log: "TorConnect: Entering Initial state" +console.log: "TorConnect: Observed profile-after-change" +console.log: "TorConnect: Observing topic 'TorProcessExited'" +console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: Observing topic 'torsettings:ready'" +console.log: "TorSettings: Observed profile-after-change" +1687903431712 Marionette INFO Marionette enabled +console.log: "TorConnect: Will load after bootstrap => [about:blank]" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:53499/devtools/browser/de12a446-2f37-4a86-958f-cff1379a68c6 +1687903433885 Marionette INFO Listening on port 53504 +1687903434216 RemoteAgent WARN TLS certificate errors will be ignored for this session +1687903637929 Marionette INFO Stopped listening on port 53504 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first. +JavaScript error: resource://gre/modules/PageThumbs.jsm, line 709: AbortError: IOUtils.profileBeforeChange getter: IOUtils: profileBeforeChange phase has already finished +1687903638316 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1687903660938 geckodriver INFO Listening on 127.0.0.1:53553 +1687903664746 mozrunner::runner INFO Running command: "C:\\Users\\\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "53554" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofilec5RJTT" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: init()" +console.log: "TorConnect: Entering Initial state" +console.log: "TorConnect: Observed profile-after-change" +console.log: "TorConnect: Observing topic 'TorProcessExited'" +console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: Observing topic 'torsettings:ready'" +console.log: "TorSettings: Observed profile-after-change" +1687903665325 Marionette INFO Marionette enabled +console.log: "TorConnect: Will load after bootstrap => [about:blank]" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:53554/devtools/browser/355ccb1b-0116-46c6-8b57-26c20c38a8a8 +1687903666934 Marionette INFO Listening on port 53559 +1687903667201 RemoteAgent WARN TLS certificate errors will be ignored for this session +SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/4336e1c3-b0a3-4fce-8d5f-788a853daf49'" nsresult: "0x805303f4 ()" location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670" data: yes] +Stack: mainThreadFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15 +mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10 +_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22 +urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17 +_resurrectSource@resource://devtools/server/actors/thread.js:2142:35 +addAllSources@resource://devtools/server/actors/thread.js:1509:14 +watch@resource://devtools/server/actors/resources/sources.js:52:17 +watchResources@resource://devtools/server/actors/resources/index.js:239:19 +_watchTargetResources@resource://devtools/server/actors/targets/target-actor-mixin.js:156:24 +addWatcherDataEntry@resource://devtools/server/actors/targets/target-actor-mixin.js:47:20 +_addWatcherDataEntry@resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm:483:24 +receiveMessage@resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm:425:21 +Line: 670, column: 0 +console.error: ({}) +SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/a3a8de7d-9171-4693-b274-22ce46ee810b'" nsresult: "0x805303f4 ()" location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670" data: yes] +Stack: mainThreadFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15 +mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10 +_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22 +urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17 +_resurrectSource@resource://devtools/server/actors/thread.js:2142:35 +addAllSources@resource://devtools/server/actors/thread.js:1509:14 +watch@resource://devtools/server/actors/resources/sources.js:52:17 +watchResources@resource://devtools/server/actors/resources/index.js:239:19 +_watchTargetResources@resource://devtools/server/actors/targets/target-actor-mixin.js:156:24 +addWatcherDataEntry@resource://devtools/server/actors/targets/target-actor-mixin.js:47:20 +_addWatcherDataEntry@resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm:483:24 +receiveMessage@resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm:425:21 +Line: 670, column: 0 +console.error: ({}) +JavaScript error: resource://devtools/shared/DevToolsUtils.js, line 670: Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/4336e1c3-b0a3-4fce-8d5f-788a853daf49' +JavaScript error: resource://devtools/shared/DevToolsUtils.js, line 670: Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/a3a8de7d-9171-4693-b274-22ce46ee810b' +console.warn: "Resource of root-node was not found." +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.warn: "Resource of root-node was not found." +SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/4336e1c3-b0a3-4fce-8d5f-788a853daf49'" nsresult: "0x805303f4 ()" location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670" data: yes] +Stack: mainThreadFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15 +mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10 +_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22 +urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17 +_resurrectSource@resource://devtools/server/actors/thread.js:2142:35 +addAllSources@resource://devtools/server/actors/thread.js:1509:14 +_onWindowReady@resource://devtools/server/actors/thread.js:1818:12 +_emit@resource://devtools/shared/event-emitter.js:226:34 +emit@resource://devtools/shared/event-emitter.js:172:18 +emit@resource://devtools/shared/event-emitter.js:324:18 +_windowReady@resource://devtools/server/actors/targets/browsing-context.js:1406:10 +DebuggerProgressListener.prototype.onWindowCreated<@resource://devtools/server/actors/targets/browsing-context.js:1761:23 +exports.makeInfallible/<@resource://devtools/shared/ThreadSafeDevToolsUtils.js:103:22 +Line: 670, column: 0 +console.error: ({}) +SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/a3a8de7d-9171-4693-b274-22ce46ee810b'" nsresult: "0x805303f4 ()" location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670" data: yes] +Stack: mainThreadFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15 +mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10 +_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22 +urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17 +_resurrectSource@resource://devtools/server/actors/thread.js:2142:35 +addAllSources@resource://devtools/server/actors/thread.js:1509:14 +_onWindowReady@resource://devtools/server/actors/thread.js:1818:12 +_emit@resource://devtools/shared/event-emitter.js:226:34 +emit@resource://devtools/shared/event-emitter.js:172:18 +emit@resource://devtools/shared/event-emitter.js:324:18 +_windowReady@resource://devtools/server/actors/targets/browsing-context.js:1406:10 +DebuggerProgressListener.prototype.onWindowCreated<@resource://devtools/server/actors/targets/browsing-context.js:1761:23 +exports.makeInfallible/<@resource://devtools/shared/ThreadSafeDevToolsUtils.js:103:22 +Line: 670, column: 0 +console.error: ({}) +JavaScript error: resource://devtools/shared/DevToolsUtils.js, line 670: Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/4336e1c3-b0a3-4fce-8d5f-788a853daf49' +JavaScript error: resource://devtools/shared/DevToolsUtils.js, line 670: Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/a3a8de7d-9171-4693-b274-22ce46ee810b' +console.error: (new TypeError("container.editor is undefined", "resource://devtools/client/inspector/markup/markup.js", 1619)) +1687904059407 Marionette INFO Stopped listening on port 53559 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first. +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofilec5RJTT\thumbnails) because it does not exist + +###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + +1687904059696 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +/gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:53581/devtools/browser/2997ead3-6598-40b5-a8b3-c01d69589a10 +1687903915090 Marionette INFO Listening on port 53586 +1687903915575 RemoteAgent WARN TLS certificate errors will be ignored for this session +1687904061948 Marionette INFO Stopped listening on port 53586 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first. +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofilemGFdBU\thumbnails) because it does not exist +[Parent 7928, IPC I/O Parent] WARNING: file /var/tmp/build/firefox-b6010b1466c9/ipc/chromium/src/base/process_util_win.cc:167 +[Parent 7928, IPC I/O Parent] WARNING: pipe error: 232: file /var/tmp/build/firefox-b6010b1466c9/ipc/chromium/src/chrome/common/ipc_channel_win.cc:544 +1687904062415 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1687904107259 geckodriver INFO Listening on 127.0.0.1:53630 +1687904110330 mozrunner::runner INFO Running command: "C:\\Users\\\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "53631" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofile8K4FD1" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: init()" +console.log: "TorConnect: Entering Initial state" +console.log: "TorConnect: Observed profile-after-change" +console.log: "TorConnect: Observing topic 'TorProcessExited'" +console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: Observing topic 'torsettings:ready'" +console.log: "TorSettings: Observed profile-after-change" +1687904110906 Marionette INFO Marionette enabled +console.log: "TorConnect: Will load after bootstrap => [about:blank]" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:53631/devtools/browser/037c4fde-6624-4d8e-9a76-ef6de40769d0 +1687904112499 Marionette INFO Listening on port 53636 +1687904112920 RemoteAgent WARN TLS certificate errors will be ignored for this session +1687904536185 Marionette INFO Stopped listening on port 53636 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first. +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofile8K4FD1\thumbnails) because it does not exist + +###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + +1687904536422 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1687984144341 geckodriver INFO Listening on 127.0.0.1:59014 +1687984148447 mozrunner::runner INFO Running command: "C:\\Users\\\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "59015" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofilePLIlxF" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: init()" +console.log: "TorConnect: Entering Initial state" +console.log: "TorConnect: Observed profile-after-change" +console.log: "TorConnect: Observing topic 'TorProcessExited'" +console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: Observing topic 'torsettings:ready'" +console.log: "TorSettings: Observed profile-after-change" +1687984149607 Marionette INFO Marionette enabled +console.log: "TorConnect: Will load after bootstrap => [about:blank]" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:59015/devtools/browser/9c846e9a-ade1-4e69-bc6e-91c4fb184cc5 +1687984152249 Marionette INFO Listening on port 50767 + +###!!! [Child][MessageChannel] Error: (msgtype=0x390097,name=PContent::Msg_InitBackground) Channel closing: too late to send/recv, messages will be lost + +1687984152500 RemoteAgent WARN TLS certificate errors will be ignored for this session +IPDL protocol error: Handler returned error code! + +###!!! [Parent][DispatchAsyncMessage] Error: PClientManager::Msg_ExpectFutureClientSource Processing error: message was deserialized, but the handler returned false (indicating failure) + +IPDL protocol error: Handler returned error code! + +###!!! [Parent][DispatchAsyncMessage] Error: PClientManager::Msg_ForgetFutureClientSource Processing error: message was deserialized, but the handler returned false (indicating failure) + +SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/e82d3d44-a580-4a44-a340-6909b635cff2'" nsresult: "0x805303f4 ()" location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670" data: yes] +Stack: mainThreadFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15 +mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10 +_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22 +urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17 +_resurrectSource@resource://devtools/server/actors/thread.js:2142:35 +addAllSources@resource://devtools/server/actors/thread.js:1509:14 +watch@resource://devtools/server/actors/resources/sources.js:52:17 +watchResources@resource://devtools/server/actors/resources/index.js:239:19 +_watchTargetResources@resource://devtools/server/actors/targets/target-actor-mixin.js:156:24 +addWatcherDataEntry@resource://devtools/server/actors/targets/target-actor-mixin.js:47:20 +_addWatcherDataEntry@resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm:483:24 +receiveMessage@resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm:425:21 +Line: 670, column: 0 +console.error: ({}) +SourceActor threw an exception: [Exception... "Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/b6ea1cf8-5449-4402-9d81-b941f13def13'" nsresult: "0x805303f4 ()" location: "JS frame :: resource://devtools/shared/DevToolsUtils.js :: mainThreadFetch/< :: line 670" data: yes] +Stack: mainThreadFetch/<@resource://devtools/shared/DevToolsUtils.js:670:15 +mainThreadFetch@resource://devtools/shared/DevToolsUtils.js:516:10 +_fetchURLContents@resource://devtools/server/actors/utils/sources-manager.js:442:22 +urlContents@resource://devtools/server/actors/utils/sources-manager.js:406:17 +_resurrectSource@resource://devtools/server/actors/thread.js:2142:35 +addAllSources@resource://devtools/server/actors/thread.js:1509:14 +watch@resource://devtools/server/actors/resources/sources.js:52:17 +watchResources@resource://devtools/server/actors/resources/index.js:239:19 +_watchTargetResources@resource://devtools/server/actors/targets/target-actor-mixin.js:156:24 +addWatcherDataEntry@resource://devtools/server/actors/targets/target-actor-mixin.js:47:20 +_addWatcherDataEntry@resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm:483:24 +receiveMessage@resource://devtools/server/connectors/js-window-actor/DevToolsFrameChild.jsm:425:21 +Line: 670, column: 0 +console.error: ({}) +JavaScript error: resource://devtools/shared/DevToolsUtils.js, line 670: Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/e82d3d44-a580-4a44-a340-6909b635cff2' +JavaScript error: resource://devtools/shared/DevToolsUtils.js, line 670: Failed to open input source 'blob:moz-extension://3064b8c5-bffd-4bf8-b2f1-210b12185538/b6ea1cf8-5449-4402-9d81-b941f13def13' +console.warn: "Resource of root-node was not found." +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +console.warn: "Resource of root-node was not found." +1687985372001 Marionette INFO Stopped listening on port 50767 +JavaScript error: resource://devtools/shared/protocol/Front.js, line 103: Error: Connection closed, pending request to server0.conn0.child4/domnode492, type setNodeValue failed + +Request stack: +request@resource://devtools/shared/protocol/Front.js:289:14 +generateRequestMethods/ [about:blank]" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:61946/devtools/browser/d0eed373-1242-4bca-b540-1364e0be7e06 +1687985426140 Marionette INFO Listening on port 61951 +1687985426557 RemoteAgent WARN TLS certificate errors will be ignored for this session +1687985625210 Marionette INFO Stopped listening on port 61951 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first. +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofilegYUPOb\thumbnails) because it does not exist + +###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + +1687985625780 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1687985688676 geckodriver INFO Listening on 127.0.0.1:55996 +1687985691344 mozrunner::runner INFO Running command: "C:\\Users\\\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "55997" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofileBmOWca" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: init()" +console.log: "TorConnect: Entering Initial state" +console.log: "TorConnect: Observed profile-after-change" +console.log: "TorConnect: Observing topic 'TorProcessExited'" +console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: Observing topic 'torsettings:ready'" +console.log: "TorSettings: Observed profile-after-change" +1687985692041 Marionette INFO Marionette enabled +console.log: "TorConnect: Will load after bootstrap => [about:blank]" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:55997/devtools/browser/dfbdb3dd-1385-4d3a-bee2-99e5d0160ee4 +1687985693944 Marionette INFO Listening on port 61841 +1687985694051 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1687986146314 Marionette INFO Stopped listening on port 61841 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first. +JavaScript error: resource://gre/modules/PageThumbs.jsm, line 709: AbortError: IOUtils.profileBeforeChange getter: IOUtils: profileBeforeChange phase has already finished + +###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + +1687986147248 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1687986152327 geckodriver INFO Listening on 127.0.0.1:56860 +1687986160441 mozrunner::runner INFO Running command: "C:\\Users\\\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "56861" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofileHNoMEO" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: init()" +console.log: "TorConnect: Entering Initial state" +console.log: "TorConnect: Observed profile-after-change" +console.log: "TorConnect: Observing topic 'TorProcessExited'" +console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: Observing topic 'torsettings:ready'" +console.log: "TorSettings: Observed profile-after-change" +1687986162579 Marionette INFO Marionette enabled +console.log: "TorConnect: Will load after bootstrap => [about:blank]" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:56861/devtools/browser/949df630-05d3-4b4e-b107-66539d4d4922 +1687986167920 Marionette INFO Listening on port 56873 +1687986168492 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1687986299930 Marionette INFO Stopped listening on port 56873 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +!!! error running onStopped callback: TypeError: callback is not a function +JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first. +JavaScript error: resource://gre/modules/PageThumbs.jsm, line 709: AbortError: IOUtils.profileBeforeChange getter: IOUtils: profileBeforeChange phase has already finished + +###!!! [Parent][MessageChannel] Error: (msgtype=0x390076,name=PContent::Msg_DestroyBrowsingContextGroup) Closed channel: cannot send/recv + + +###!!! [Child][MessageChannel] Error: (msgtype=0x23002E,name=PBrowser::Msg___delete__) Channel closing: too late to send/recv, messages will be lost + +1687986301187 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +1688064782612 geckodriver INFO Listening on 127.0.0.1:53074 +1688064786507 mozrunner::runner INFO Running command: "C:\\Users\\\\Helium\\Desktop\\Tor Browser\\Browser\\firefox.exe" "--marionette" "--remote-debugging-port" "53075" "--remote-allow-hosts" "localhost" "-no-remote" "-profile" "C:\\Users\\Helium\\AppData\\Local\\Temp\\rust_mozprofilee4dopq" +console.log: "TorSettings: loadFromPrefs()" +console.log: "TorConnect: init()" +console.log: "TorConnect: Entering Initial state" +console.log: "TorConnect: Observed profile-after-change" +console.log: "TorConnect: Observing topic 'TorProcessExited'" +console.log: "TorConnect: Observing topic 'TorLogHasWarnOrErr'" +console.log: "TorConnect: Observing topic 'torsettings:ready'" +console.log: "TorSettings: Observed profile-after-change" +1688064787228 Marionette INFO Marionette enabled +console.log: "TorConnect: Will load after bootstrap => [about:blank]" +console.error: "Could not load engine blockchair-onion@search.mozilla.org: Error: Extension is invalid" +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XPCOMUtils.jsm, line 161: TypeError: Cc[aContract] is undefined +DevTools listening on ws://localhost:53075/devtools/browser/f975a788-4cc3-4b93-8ff4-00ebb08433d1 +1688064788880 Marionette INFO Listening on port 52296 +1688064788914 RemoteAgent WARN TLS certificate errors will be ignored for this session +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofilee4dopq\thumbnails) because it does not exist +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +JavaScript error: resource://gre/modules/XULStore.jsm, line 66: Error: Can't find profile directory. +1688065043692 Marionette INFO Stopped listening on port 52296 +JavaScript error: resource:///modules/Interactions.jsm, line 209: NS_ERROR_FAILURE: Component returned failure code: 0x80004005 (NS_ERROR_FAILURE) [nsIUserIdleService.removeIdleObserver] +JavaScript error: chrome://remote/content/marionette/cert.js, line 55: NS_ERROR_NOT_AVAILABLE: Component returned failure code: 0x80040111 (NS_ERROR_NOT_AVAILABLE) [nsICertOverrideService.setDisableAllSecurityChecksAndLetAttackersInterceptMyData] +!!! error running onStopped callback: TypeError: callback is not a function +JavaScript error: resource:///modules/sessionstore/SessionFile.jsm, line 375: Error: _initWorker called too early! Please read the session file from disk first. +JavaScript error: resource://gre/modules/PromiseWorker.jsm, line 106: Error: Could not get children of file(C:\Users\Helium\AppData\Local\Temp\rust_mozprofilee4dopq\thumbnails) because it does not exist +[Parent 6908, IPC I/O Parent] WARNING: file /var/tmp/build/firefox-b6010b1466c9/ipc/chromium/src/base/process_util_win.cc:167 + +###!!! [Parent][RunMessage] Error: Channel closing: too late to send/recv, messages will be lost + +1688065044075 RemoteAgent ERROR unable to stop listener: [Exception... "Component returned failure code: 0x8000ffff (NS_ERROR_UNEXPECTED) [nsIWindowMediator.getEnumerator]" nsresult: "0x8000ffff (NS_ERROR_UNEXPECTED)" location: "JS frame :: chrome://remote/content/cdp/observers/TargetObserver.jsm :: stop :: line 64" data: no] Stack trace: stop()@TargetObserver.jsm:64 +unwatchForTabs()@TargetList.jsm:70 +unwatchForTargets()@TargetList.jsm:37 +destructor()@TargetList.jsm:109 +stop()@CDP.jsm:104 +close()@RemoteAgent.jsm:138 +>>>>>>> 98de158ee6f14b35b2a5ac61427d7de36b4ee442 diff --git a/MarketPlaces/Initialization/markets_mining.py b/MarketPlaces/Initialization/markets_mining.py index c097411..aad5f63 100644 --- a/MarketPlaces/Initialization/markets_mining.py +++ b/MarketPlaces/Initialization/markets_mining.py @@ -21,10 +21,11 @@ from MarketPlaces.CityMarket.crawler_selenium import crawler as crawlerCityMarke from MarketPlaces.DarkMatter.crawler_selenium import crawler as crawlerDarkMatter from MarketPlaces.M00nkeyMarket.crawler_selenium import crawler as crawlerM00nkeyMarket - - +import configparser import time +config = configparser.ConfigParser() +config.read('../../setup.ini') CURRENT_DATE = str("%02d" % date.today().month) + str("%02d" % date.today().day) + str("%04d" % date.today().year) @@ -40,7 +41,8 @@ def getMarkets(): def createDirectory(mkt): # Package should already be there, holding crawler and parser - pagesDir = '../' + mkt + '/HTML_Pages' + # pagesDir = '../' + mkt + '/HTML_Pages' + pagesDir = os.path.join(config.get('Project', 'shared_folder'), "MarketPlaces/" + mkt + "/HTML_Pages") if not os.path.isdir(pagesDir): os.makedirs(pagesDir) diff --git a/MarketPlaces/Initialization/prepare_parser.py b/MarketPlaces/Initialization/prepare_parser.py index de13899..accd697 100644 --- a/MarketPlaces/Initialization/prepare_parser.py +++ b/MarketPlaces/Initialization/prepare_parser.py @@ -71,7 +71,7 @@ def persist_data(url, row, cur): def new_parse(marketPlace, url, createLog): - from MarketPlaces.Initialization.markets_mining import CURRENT_DATE + from MarketPlaces.Initialization.markets_mining import config, CURRENT_DATE print("Parsing the " + marketPlace + " marketplace and conduct data classification to store the information in the database.") @@ -100,12 +100,14 @@ def new_parse(marketPlace, url, createLog): " in the _Logs folder to read files from this Market Place of this date again.") raise SystemExit + mainDir = os.path.join(config.get('Project', 'shared_folder'), "MarketPlaces/" + marketPlace + "/HTML_Pages") + # Reading the Listing Html Pages - for fileListing in glob.glob(os.path.join("..\\" + marketPlace + "\\HTML_Pages\\" + CURRENT_DATE + "\\Listing", '*.html')): + for fileListing in glob.glob(os.path.join(mainDir, CURRENT_DATE + "\\Listing", '*.html')): lines.append(fileListing) # Reading the Description Html Pages - for fileDescription in glob.glob(os.path.join("..\\" + marketPlace + "\\HTML_Pages\\" + CURRENT_DATE + "\\Description", '*.html')): + for fileDescription in glob.glob(os.path.join(mainDir, CURRENT_DATE + "\\Description", '*.html')): lns.append(fileDescription) # Parsing the Description Pages and put the tag's content into a dictionary (Hash table) diff --git a/MarketPlaces/LionMarketplace/crawler_selenium.py b/MarketPlaces/LionMarketplace/crawler_selenium.py index 5212aad..3310aca 100644 --- a/MarketPlaces/LionMarketplace/crawler_selenium.py +++ b/MarketPlaces/LionMarketplace/crawler_selenium.py @@ -24,8 +24,6 @@ from MarketPlaces.Initialization.prepare_parser import new_parse from MarketPlaces.LionMarketplace.parser import lionmarketplace_links_parser from MarketPlaces.Utilities.utilities import cleanHTML -config = configparser.ConfigParser() -config.read('../../setup.ini') counter = 1 baseURL = 'http://lionznqc2hg2wsp5vgruqait4cpknihwlje6hkjyi52lcl5ivyf7bcad.onion/' @@ -51,6 +49,8 @@ def startCrawling(): # Opens Tor Browser #prompts for ENTER input to continue def opentor(): + from MarketPlaces.Initialization.markets_mining import config + global pid print("Connecting Tor...") pro = subprocess.Popen(config.get('TOR', 'firefox_binary_path')) @@ -89,6 +89,8 @@ def closetor(driver): # Creates FireFox 'driver' and configure its 'Profile' # to use Tor proxy and socket def createFFDriver(): + from MarketPlaces.Initialization.markets_mining import config + ff_binary = FirefoxBinary(config.get('TOR', 'firefox_binary_path')) ff_prof = FirefoxProfile(config.get('TOR', 'firefox_profile_path')) @@ -154,12 +156,14 @@ def savePage(page, url): # Gets the full path of the page to be saved along with its appropriate file name #@param: raw url as crawler crawls through every site def getFullPathName(url): - from MarketPlaces.Initialization.markets_mining import CURRENT_DATE + from MarketPlaces.Initialization.markets_mining import config, CURRENT_DATE + + mainDir = os.path.join(config.get('Project', 'shared_folder'), "MarketPlaces/" + getMKTName() + "/HTML_Pages") fileName = getNameFromURL(url) if isDescriptionLink(url): - fullPath = r'..\LionMarketplace\HTML_Pages\\' + CURRENT_DATE + r'\\Description\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Description\\' + fileName + '.html') else: - fullPath = r'..\LionMarketplace\HTML_Pages\\' + CURRENT_DATE + r'\\Listing\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Listing\\' + fileName + '.html') return fullPath diff --git a/MarketPlaces/M00nkeyMarket/crawler_selenium.py b/MarketPlaces/M00nkeyMarket/crawler_selenium.py index 92898dd..1a4776e 100644 --- a/MarketPlaces/M00nkeyMarket/crawler_selenium.py +++ b/MarketPlaces/M00nkeyMarket/crawler_selenium.py @@ -25,8 +25,6 @@ from MarketPlaces.Initialization.prepare_parser import new_parse from MarketPlaces.M00nkeyMarket.parser import m00nkey_links_parser from MarketPlaces.Utilities.utilities import cleanHTML -config = configparser.ConfigParser() -config.read('../../setup.ini') counter = 1 baseURL = 'http://moonkey4f2mkcp6hpackeea356puiry27h3dz3hzbt3adbmsk4gs7wyd.onion/' @@ -52,6 +50,8 @@ def startCrawling(): # Opens Tor Browser #prompts for ENTER input to continue def opentor(): + from MarketPlaces.Initialization.markets_mining import config + global pid print("Connecting Tor...") pro = subprocess.Popen(config.get('TOR', 'firefox_binary_path')) @@ -90,6 +90,8 @@ def closetor(driver): # Creates FireFox 'driver' and configure its 'Profile' # to use Tor proxy and socket def createFFDriver(): + from MarketPlaces.Initialization.markets_mining import config + ff_binary = FirefoxBinary(config.get('TOR', 'firefox_binary_path')) ff_prof = FirefoxProfile(config.get('TOR', 'firefox_profile_path')) @@ -170,12 +172,14 @@ def savePage(page, url): # Gets the full path of the page to be saved along with its appropriate file name #@param: raw url as crawler crawls through every site def getFullPathName(url): - from MarketPlaces.Initialization.markets_mining import CURRENT_DATE + from MarketPlaces.Initialization.markets_mining import config, CURRENT_DATE + + mainDir = os.path.join(config.get('Project', 'shared_folder'), "MarketPlaces/" + getMKTName() + "/HTML_Pages") fileName = getNameFromURL(url) if isDescriptionLink(url): - fullPath = r'..\M00nkeyMarket\HTML_Pages\\' + CURRENT_DATE + r'\\Description\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Description\\' + fileName + '.html') else: - fullPath = r'..\M00nkeyMarket\HTML_Pages\\' + CURRENT_DATE + r'\\Listing\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Listing\\' + fileName + '.html') return fullPath diff --git a/MarketPlaces/MikesGrandStore/crawler_selenium.py b/MarketPlaces/MikesGrandStore/crawler_selenium.py index 1631e7b..cd5af50 100644 --- a/MarketPlaces/MikesGrandStore/crawler_selenium.py +++ b/MarketPlaces/MikesGrandStore/crawler_selenium.py @@ -24,8 +24,6 @@ from MarketPlaces.Initialization.prepare_parser import new_parse from MarketPlaces.MikesGrandStore.parser import mikesgrandstore_links_parser from MarketPlaces.Utilities.utilities import cleanHTML -config = configparser.ConfigParser() -config.read('../../setup.ini') counter = 1 baseURL = 'http://4yx2akutmkhwfgzlpdxiah7cknurw6vlddlq24fxa3r3ebophwgpvhyd.onion/' @@ -51,6 +49,8 @@ def startCrawling(): # Opens Tor Browser #prompts for ENTER input to continue def opentor(): + from MarketPlaces.Initialization.markets_mining import config + global pid print("Connecting Tor...") pro = subprocess.Popen(config.get('TOR', 'firefox_binary_path')) @@ -89,6 +89,8 @@ def closetor(driver): # Creates FireFox 'driver' and configure its 'Profile' # to use Tor proxy and socket def createFFDriver(): + from MarketPlaces.Initialization.markets_mining import config + ff_binary = FirefoxBinary(config.get('TOR', 'firefox_binary_path')) ff_prof = FirefoxProfile(config.get('TOR', 'firefox_profile_path')) @@ -153,12 +155,14 @@ def savePage(page, url): # Gets the full path of the page to be saved along with its appropriate file name #@param: raw url as crawler crawls through every site def getFullPathName(url): - from MarketPlaces.Initialization.markets_mining import CURRENT_DATE + from MarketPlaces.Initialization.markets_mining import config, CURRENT_DATE + + mainDir = os.path.join(config.get('Project', 'shared_folder'), "MarketPlaces/" + getMKTName() + "/HTML_Pages") fileName = getNameFromURL(url) if isDescriptionLink(url): - fullPath = r'..\MikesGrandStore\HTML_Pages\\' + CURRENT_DATE + r'\\Description\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Description\\' + fileName + '.html') else: - fullPath = r'..\MikesGrandStore\HTML_Pages\\' + CURRENT_DATE + r'\\Listing\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Listing\\' + fileName + '.html') return fullPath diff --git a/MarketPlaces/ThiefWorld/crawler_selenium.py b/MarketPlaces/ThiefWorld/crawler_selenium.py index a8293bb..5478763 100644 --- a/MarketPlaces/ThiefWorld/crawler_selenium.py +++ b/MarketPlaces/ThiefWorld/crawler_selenium.py @@ -24,8 +24,6 @@ from MarketPlaces.Initialization.prepare_parser import new_parse from MarketPlaces.ThiefWorld.parser import thiefworld_links_parser from MarketPlaces.Utilities.utilities import cleanHTML -config = configparser.ConfigParser() -config.read('../../setup.ini') counter = 1 baseURL = 'http://qsw7iurcrdwyml5kg4oxbmtqrcnpxiag3iumdarefzeunnyc2dnyljad.onion/' @@ -33,24 +31,26 @@ baseURL = 'http://qsw7iurcrdwyml5kg4oxbmtqrcnpxiag3iumdarefzeunnyc2dnyljad.onion # Opens Tor Browser, crawls the website, then parses, then closes tor #acts like the main method for the crawler, another function at the end of this code calls this function later def startCrawling(): - opentor() - # mktName = getMKTName() - driver = getAccess() + # opentor() + mktName = getMKTName() + # driver = getAccess() + # + # if driver != 'down': + # try: + # login(driver) + # crawlForum(driver) + # except Exception as e: + # print(driver.current_url, e) + # closetor(driver) - if driver != 'down': - try: - login(driver) - crawlForum(driver) - except Exception as e: - print(driver.current_url, e) - closetor(driver) - - # new_parse(forumName, baseURL, False) + new_parse(mktName, baseURL, False) # Opens Tor Browser #prompts for ENTER input to continue def opentor(): + from MarketPlaces.Initialization.markets_mining import config + global pid print("Connecting Tor...") pro = subprocess.Popen(config.get('TOR', 'firefox_binary_path')) @@ -89,6 +89,8 @@ def closetor(driver): # Creates FireFox 'driver' and configure its 'Profile' # to use Tor proxy and socket def createFFDriver(): + from MarketPlaces.Initialization.markets_mining import config + ff_binary = FirefoxBinary(config.get('TOR', 'firefox_binary_path')) ff_prof = FirefoxProfile(config.get('TOR', 'firefox_profile_path')) @@ -161,12 +163,14 @@ def savePage(page, url): # Gets the full path of the page to be saved along with its appropriate file name #@param: raw url as crawler crawls through every site def getFullPathName(url): - from MarketPlaces.Initialization.markets_mining import CURRENT_DATE + from MarketPlaces.Initialization.markets_mining import config, CURRENT_DATE + + mainDir = os.path.join(config.get('Project', 'shared_folder'), "MarketPlaces/" + getMKTName() + "/HTML_Pages") fileName = getNameFromURL(url) if isDescriptionLink(url): - fullPath = r'..\ThiefWorld\HTML_Pages\\' + CURRENT_DATE + r'\\Description\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Description\\' + fileName + '.html') else: - fullPath = r'..\ThiefWorld\HTML_Pages\\' + CURRENT_DATE + r'\\Listing\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Listing\\' + fileName + '.html') return fullPath diff --git a/MarketPlaces/Tor2door/crawler_selenium.py b/MarketPlaces/Tor2door/crawler_selenium.py index baef719..b0c5a6b 100644 --- a/MarketPlaces/Tor2door/crawler_selenium.py +++ b/MarketPlaces/Tor2door/crawler_selenium.py @@ -23,8 +23,6 @@ from MarketPlaces.Initialization.prepare_parser import new_parse from MarketPlaces.Tor2door.parser import tor2door_links_parser from MarketPlaces.Utilities.utilities import cleanHTML -config = configparser.ConfigParser() -config.read('../../setup.ini') counter = 1 baseURL = 'http://yzrrne3pveltulbavydr2kiashvlnysdwclwmklo6cyjuqpxi7ku4xqd.onion' @@ -48,6 +46,8 @@ def startCrawling(): # Opens Tor Browser def opentor(): + from MarketPlaces.Initialization.markets_mining import config + global pid print("Connecting Tor...") pro = subprocess.Popen(config.get('TOR', 'firefox_binary_path')) @@ -131,6 +131,8 @@ def closetor(driver): # Creates FireFox 'driver' and configure its 'Profile' # to use Tor proxy and socket def createFFDriver(): + from MarketPlaces.Initialization.markets_mining import config + ff_binary = FirefoxBinary(config.get('TOR', 'firefox_binary_path')) ff_prof = FirefoxProfile(config.get('TOR', 'firefox_profile_path')) @@ -184,12 +186,14 @@ def savePage(page, url): # Gets the full path of the page to be saved along with its appropriate file name def getFullPathName(url): - from MarketPlaces.Initialization.markets_mining import CURRENT_DATE + from MarketPlaces.Initialization.markets_mining import config, CURRENT_DATE + + mainDir = os.path.join(config.get('Project', 'shared_folder'), "MarketPlaces/" + getMKTName() + "/HTML_Pages") fileName = getNameFromURL(url) if isDescriptionLink(url): - fullPath = r'..\Tor2door\HTML_Pages\\' + CURRENT_DATE + r'\\Description\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Description\\' + fileName + '.html') else: - fullPath = r'..\Tor2door\HTML_Pages\\' + CURRENT_DATE + r'\\Listing\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Listing\\' + fileName + '.html') return fullPath diff --git a/MarketPlaces/TorBay/crawler_selenium.py b/MarketPlaces/TorBay/crawler_selenium.py index 3c0619e..cadc72c 100644 --- a/MarketPlaces/TorBay/crawler_selenium.py +++ b/MarketPlaces/TorBay/crawler_selenium.py @@ -25,8 +25,6 @@ from MarketPlaces.Initialization.prepare_parser import new_parse from MarketPlaces.TorBay.parser import torbay_links_parser from MarketPlaces.Utilities.utilities import cleanHTML -config = configparser.ConfigParser() -config.read('../../setup.ini') counter = 1 baseURL = 'http://torbay3253zck4ym5cbowwvrbfjjzruzthrx3np5y6owvifrnhy5ybid.onion/' @@ -52,6 +50,8 @@ def startCrawling(): # Opens Tor Browser #prompts for ENTER input to continue def opentor(): + from MarketPlaces.Initialization.markets_mining import config + global pid print("Connecting Tor...") pro = subprocess.Popen(config.get('TOR', 'firefox_binary_path')) @@ -90,6 +90,8 @@ def closetor(driver): # Creates FireFox 'driver' and configure its 'Profile' # to use Tor proxy and socket def createFFDriver(): + from MarketPlaces.Initialization.markets_mining import config + ff_binary = FirefoxBinary(config.get('TOR', 'firefox_binary_path')) ff_prof = FirefoxProfile(config.get('TOR', 'firefox_profile_path')) @@ -154,15 +156,14 @@ def savePage(page, url): # Gets the full path of the page to be saved along with its appropriate file name #@param: raw url as crawler crawls through every site def getFullPathName(url): + from MarketPlaces.Initialization.markets_mining import config, CURRENT_DATE + + mainDir = os.path.join(config.get('Project', 'shared_folder'), "MarketPlaces/" + getMKTName() + "/HTML_Pages") fileName = getNameFromURL(url) if isDescriptionLink(url): - fullPath = r'..\TorBay\HTML_Pages\\' + str( - "%02d" % date.today().month) + str("%02d" % date.today().day) + str( - "%04d" % date.today().year) + r'\\' + r'Description\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Description\\' + fileName + '.html') else: - fullPath = r'..\TorBay\HTML_Pages\\' + str( - "%02d" % date.today().month) + str("%02d" % date.today().day) + str( - "%04d" % date.today().year) + r'\\' + r'Listing\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Listing\\' + fileName + '.html') return fullPath diff --git a/MarketPlaces/TorMarket/crawler_selenium.py b/MarketPlaces/TorMarket/crawler_selenium.py index 91c2f84..35be864 100644 --- a/MarketPlaces/TorMarket/crawler_selenium.py +++ b/MarketPlaces/TorMarket/crawler_selenium.py @@ -24,8 +24,6 @@ from MarketPlaces.Initialization.prepare_parser import new_parse from MarketPlaces.TorMarket.parser import tormarket_links_parser from MarketPlaces.Utilities.utilities import cleanHTML -config = configparser.ConfigParser() -config.read('../../setup.ini') counter = 1 baseURL = 'http://22222253ebafysmwyrl4uxfcs2xm6k7zb4xyse2csne73atvxu53gfad.onion/' @@ -51,6 +49,8 @@ def startCrawling(): # Opens Tor Browser #prompts for ENTER input to continue def opentor(): + from MarketPlaces.Initialization.markets_mining import config + global pid print("Connecting Tor...") pro = subprocess.Popen(config.get('TOR', 'firefox_binary_path')) @@ -89,6 +89,8 @@ def closetor(driver): # Creates FireFox 'driver' and configure its 'Profile' # to use Tor proxy and socket def createFFDriver(): + from MarketPlaces.Initialization.markets_mining import config + ff_binary = FirefoxBinary(config.get('TOR', 'firefox_binary_path')) ff_prof = FirefoxProfile(config.get('TOR', 'firefox_profile_path')) @@ -153,12 +155,14 @@ def savePage(page, url): # Gets the full path of the page to be saved along with its appropriate file name #@param: raw url as crawler crawls through every site def getFullPathName(url): - from MarketPlaces.Initialization.markets_mining import CURRENT_DATE + from MarketPlaces.Initialization.markets_mining import config, CURRENT_DATE + + mainDir = os.path.join(config.get('Project', 'shared_folder'), "MarketPlaces/" + getMKTName() + "/HTML_Pages") fileName = getNameFromURL(url) if isDescriptionLink(url): - fullPath = r'..\TorMarket\HTML_Pages\\' + CURRENT_DATE + r'\\Description\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Description\\' + fileName + '.html') else: - fullPath = r'..\TorMarket\HTML_Pages\\' + CURRENT_DATE + r'\\Listing\\' + fileName + '.html' + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Listing\\' + fileName + '.html') return fullPath diff --git a/setup.ini b/setup.ini index 25ac704..2659ef2 100644 --- a/setup.ini +++ b/setup.ini @@ -1,11 +1,12 @@ + [TOR] firefox_binary_path = C:\\Users\\minhkhoitran\\Desktop\\Tor Browser\\Browser\\firefox.exe firefox_profile_path = C:\\Users\\minhkhoitran\\Desktop\\Tor Browser\\Browser\\TorBrowser\\Data\\Browser\\profile.default geckodriver_path = C:\\NSF-REU\\dw_pipeline_test\\selenium\\geckodriver.exe [Project] -project_directory = C:\\NSF-REU\\dw_pipeline_test -shared_folder = \\VBoxSvr\\VM_Files_(shared) +project_directory = C:\Users\Helium\\PycharmProjects\dw_pipeline_test +shared_folder = \\VBoxSvr\Shared [PostgreSQL] ip = localhost