__author__ = 'DarkWeb' ''' HiddenMarket Market Crawler (Selenium) ''' from selenium import webdriver from selenium.common.exceptions import NoSuchElementException from selenium.webdriver.firefox.firefox_profile import FirefoxProfile from selenium.webdriver.firefox.firefox_binary import FirefoxBinary from selenium.webdriver.firefox.service import Service from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.ui import WebDriverWait from PIL import Image import urllib.parse as urlparse import os, re, time import subprocess import configparser from bs4 import BeautifulSoup from MarketPlaces.Initialization.prepare_parser import new_parse from MarketPlaces.HiddenMarket.parser import hiddenmarket_links_parser from MarketPlaces.Utilities.utilities import cleanHTML counter = 1 baseURL = 'http://mipx6eedtsvfgfcmbm3utjedgjez2w4dzjdrbhtd2mt3cicwhhzspxqd.onion/' # Opens Tor Browser, crawls the website def startCrawling(): opentor() marketName = getMKTName() driver = getAccess() if driver != 'down': try: login(driver) crawlForum(driver) except Exception as e: print(driver.current_url, e) closetor(driver) new_parse(marketName, baseURL, False) # Opens Tor Browser def opentor(): from MarketPlaces.Initialization.markets_mining import config global pid print("Connecting Tor...") pro = subprocess.Popen(config.get('TOR', 'firefox_binary_path')) pid = pro.pid time.sleep(7.5) input('Tor Connected. Press ENTER to continue\n') return # Login using premade account credentials and do login captcha manually def login(driver): # wait for login page WebDriverWait(driver, 100).until(EC.visibility_of_element_located( (By.XPATH, "/html/body/div[3]/div[3]"))) # entering username and password into input boxes # usernameBox = driver.find_element(by=By.XPATH, value='//*[@id="username"]') # Username here # usernameBox.send_keys('ct1234') # passwordBox = driver.find_element(by=By.XPATH, value='//*[@id="password"]') # Password here # passwordBox.send_keys('DementedBed1230') ''' # wait for captcha page show up WebDriverWait(driver, 100).until(EC.visibility_of_element_located( (By.XPATH, "/html/body/main/div/div/div/div/div/form/div[3]/div/div[1]/label/img"))) # save captcha to local driver.find_element(by=By.XPATH, value='/html/body/main/div/div/div/div/div/form/div[3]/div/div[1]/label/img').screenshot( r'..\captcha.png') # This method will show image in any image viewer im = Image.open(r'..\captcha.png') im.show() # wait until input space show up inputBox = driver.find_element(by=By.XPATH, value='//*[@id="captcha"]') # ask user input captcha solution in terminal userIn = input("Enter solution: ") # send user solution into the input space inputBox.send_keys(userIn) # click the verify(submit) button driver.find_element(by=By.XPATH, value="/html/body/main/div/div/div/div/div/form/div[4]/button").click() ''' # input("Press ENTER when CAPTCHA is completed\n") # wait for listing page show up (This Xpath may need to change based on different seed url) # WebDriverWait(driver, 100).until(EC.visibility_of_element_located( # (By.XPATH, '/html/body/main/div/div/div[1]/div/div[1]/div/h5'))) # Returns the name of the website def getMKTName(): name = 'HiddenMarket' return name # Return the link of the website def getFixedURL(): url = 'http://mipx6eedtsvfgfcmbm3utjedgjez2w4dzjdrbhtd2mt3cicwhhzspxqd.onion/' return url # Closes Tor Browser def closetor(driver): # global pid # os.system("taskkill /pid " + str(pro.pid)) # os.system("taskkill /t /f /im tor.exe") print('Closing Tor...') driver.quit() time.sleep(3) return # Creates FireFox 'driver' and configure its 'Profile' # to use Tor proxy and socket def createFFDriver(): from MarketPlaces.Initialization.markets_mining import config ff_binary = FirefoxBinary(config.get('TOR', 'firefox_binary_path')) ff_prof = FirefoxProfile(config.get('TOR', 'firefox_profile_path')) # ff_prof.set_preference("places.history.enabled", False) # ff_prof.set_preference("privacy.clearOnShutdown.offlineApps", True) # ff_prof.set_preference("privacy.clearOnShutdown.passwords", True) # ff_prof.set_preference("privacy.clearOnShutdown.siteSettings", True) # ff_prof.set_preference("privacy.sanitize.sanitizeOnShutdown", True) # ff_prof.set_preference("signon.rememberSignons", False) # ff_prof.set_preference("network.cookie.lifetimePolicy", 2) # ff_prof.set_preference("network.dns.disablePrefetch", True) # ff_prof.set_preference("network.http.sendRefererHeader", 0) # ff_prof.set_preference("permissions.default.image", 3) # ff_prof.set_preference("browser.download.folderList", 2) # ff_prof.set_preference("browser.download.manager.showWhenStarting", False) # ff_prof.set_preference("browser.helperApps.neverAsk.saveToDisk", "text/plain") ff_prof.set_preference('network.proxy.type', 1) ff_prof.set_preference("network.proxy.socks_version", 5) ff_prof.set_preference('network.proxy.socks', '127.0.0.1') ff_prof.set_preference('network.proxy.socks_port', 9150) ff_prof.set_preference('network.proxy.socks_remote_dns', True) ff_prof.set_preference("javascript.enabled", False) ff_prof.update_preferences() service = Service(config.get('TOR', 'geckodriver_path')) driver = webdriver.Firefox(firefox_binary=ff_binary, firefox_profile=ff_prof, service=service) return driver def getAccess(): url = getFixedURL() driver = createFFDriver() try: driver.get(url) return driver except: driver.close() return 'down' # Saves the crawled html page def savePage(page, url): cleanPage = cleanHTML(page) filePath = getFullPathName(url) os.makedirs(os.path.dirname(filePath), exist_ok=True) open(filePath, 'wb').write(cleanPage.encode('utf-8')) return # Gets the full path of the page to be saved along with its appropriate file name def getFullPathName(url): from MarketPlaces.Initialization.markets_mining import config, CURRENT_DATE mainDir = os.path.join(config.get('Project', 'shared_folder'), "MarketPlaces/" + getMKTName() + "/HTML_Pages") fileName = getNameFromURL(url) if isDescriptionLink(url): fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Description\\' + fileName + '.html') else: fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Listing\\' + fileName + '.html') return fullPath # Creates the file name from passed URL def getNameFromURL(url): global counter name = ''.join(e for e in url if e.isalnum()) if name == '': name = str(counter) counter = counter + 1 return name def getInterestedLinks(): links = [] # # Civil Software links.append('http://mipx6eedtsvfgfcmbm3utjedgjez2w4dzjdrbhtd2mt3cicwhhzspxqd.onion/category/civil_softwares') # # Tutorials - Carding links.append('http://mipx6eedtsvfgfcmbm3utjedgjez2w4dzjdrbhtd2mt3cicwhhzspxqd.onion/category/carding') # # Digital - Hacks links.append('http://mipx6eedtsvfgfcmbm3utjedgjez2w4dzjdrbhtd2mt3cicwhhzspxqd.onion/category/hacks') # Digital - Exploit Kit links.append('http://mipx6eedtsvfgfcmbm3utjedgjez2w4dzjdrbhtd2mt3cicwhhzspxqd.onion/category/exploit_kit') # # 0Day links.append('http://mipx6eedtsvfgfcmbm3utjedgjez2w4dzjdrbhtd2mt3cicwhhzspxqd.onion/category/0day') # Digital Forensics links.append('http://mipx6eedtsvfgfcmbm3utjedgjez2w4dzjdrbhtd2mt3cicwhhzspxqd.onion/category/digital_forensics') # Tutorials - Mining links.append('http://mipx6eedtsvfgfcmbm3utjedgjez2w4dzjdrbhtd2mt3cicwhhzspxqd.onion/category/mining') # Tutorials - Worms links.append('http://mipx6eedtsvfgfcmbm3utjedgjez2w4dzjdrbhtd2mt3cicwhhzspxqd.onion/category/worms') # Tutorials - Viruses links.append('http://mipx6eedtsvfgfcmbm3utjedgjez2w4dzjdrbhtd2mt3cicwhhzspxqd.onion/category/viruses') # Tutorials - Trojans links.append('http://mipx6eedtsvfgfcmbm3utjedgjez2w4dzjdrbhtd2mt3cicwhhzspxqd.onion/category/trojans') # Tutorials - Botnets links.append('http://mipx6eedtsvfgfcmbm3utjedgjez2w4dzjdrbhtd2mt3cicwhhzspxqd.onion/category/botnets') return links def crawlForum(driver): print("Crawling the HiddenMarket market") linksToCrawl = getInterestedLinks() i = 0 while i < len(linksToCrawl): link = linksToCrawl[i] print('Crawling :', link) categoryLink = link maxNumPages = 0 # temp value. try: has_next_page = True count = 0 pageCount = 1 while has_next_page: try: driver.get(link) if pageCount == 1: maxNumPages = int(driver.find_element(by=By.CLASS_NAME, value='main') .find_element(by=By.CLASS_NAME, value='pages') .find_elements(By.CLASS_NAME, value='page')[-1].text) except: driver.refresh() html = driver.page_source savePage(html, link) list = productPages(html) for item in list: itemURL = urlparse.urljoin(baseURL, str(item)) try: driver.get(itemURL) except: driver.refresh() savePage(driver.page_source, item) driver.back() # comment out # break # comment out # if count == 2: # break try: pageCount += 1 if pageCount > maxNumPages: raise NoSuchElementException pageLink = "/" + str(pageCount) + "/" link = categoryLink + pageLink count += 1 except NoSuchElementException: has_next_page = False except Exception as e: print(link, e) i += 1 input("Crawling HiddenMarket market done sucessfully. Press ENTER to continue\n") # Returns 'True' if the link is Topic link def isDescriptionLink(url): if 'product' in url: return True return False # Returns True if the link is a listingPage link def isListingLink(url): if 'category' in url: return True return False # calling the parser to define the links def productPages(html): soup = BeautifulSoup(html, "html.parser") return hiddenmarket_links_parser(soup) def crawler(): startCrawling() # print("Crawling and Parsing BestCardingWorld .... DONE!")