|
|
@ -0,0 +1,352 @@ |
|
|
|
__author__ = 'DarkWeb' |
|
|
|
|
|
|
|
''' |
|
|
|
Royal Marketplace Crawler (Selenium) |
|
|
|
''' |
|
|
|
|
|
|
|
from selenium import webdriver |
|
|
|
from selenium.webdriver.support.select import Select |
|
|
|
from selenium.common.exceptions import NoSuchElementException |
|
|
|
from selenium.webdriver.firefox.firefox_profile import FirefoxProfile |
|
|
|
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary |
|
|
|
from selenium.webdriver.firefox.service import Service |
|
|
|
from selenium.webdriver.common.by import By |
|
|
|
from selenium.webdriver.support import expected_conditions as EC |
|
|
|
from selenium.webdriver.support.ui import WebDriverWait |
|
|
|
from PIL import Image |
|
|
|
|
|
|
|
|
|
|
|
import urllib.parse as urlparse |
|
|
|
import os, re, time |
|
|
|
from datetime import date |
|
|
|
import subprocess |
|
|
|
from bs4 import BeautifulSoup |
|
|
|
from MarketPlaces.Initialization.prepare_parser import new_parse |
|
|
|
from MarketPlaces.TheDarkMarket.parser import darkmarket_links_parser |
|
|
|
from MarketPlaces.Utilities.utilities import cleanHTML |
|
|
|
|
|
|
|
counter = 1 |
|
|
|
baseURL = 'http://dark3xolguutzr2cn5twjyu6c3db2z3ai3aqyqascml5cdrleh3s2hqd.onion/' |
|
|
|
|
|
|
|
|
|
|
|
# Opens Tor Browser, crawls the website |
|
|
|
def startCrawling(): |
|
|
|
marketName = getMarketName() |
|
|
|
driver = getAccess() |
|
|
|
|
|
|
|
if driver != 'down': |
|
|
|
try: |
|
|
|
crawlForum(driver) |
|
|
|
except Exception as e: |
|
|
|
print(driver.current_url, e) |
|
|
|
closeDriver(driver) |
|
|
|
|
|
|
|
new_parse(marketPlace=marketName, url=baseURL, createLog=True) |
|
|
|
|
|
|
|
|
|
|
|
def captcha(driver): |
|
|
|
''' |
|
|
|
# wait for captcha page |
|
|
|
WebDriverWait(driver, 100).until(EC.visibility_of_element_located( |
|
|
|
(By.XPATH, "/html/body/div[2]/div/div/div/div/form/div/div[2]/button"))) |
|
|
|
|
|
|
|
inputChars = driver.find_element(by=By.XPATH, value='/html/body/div[2]/div/div/div/div/form/div/div[2]/div[1]/input') |
|
|
|
inputNum = driver.find_element(by=By.XPATH, value='/html/body/div[2]/div/div/div/div/form/div/div[2]/div[2]/input') |
|
|
|
|
|
|
|
driver.find_element(by=By.XPATH, value='/html/body/div[2]/div/div/div/div/form/div/div[1]/div/div').screenshot( |
|
|
|
r'..\Royal\captcha1.png') |
|
|
|
|
|
|
|
im = Image.open(r'..\Royal\captcha1.png') |
|
|
|
im.show() |
|
|
|
|
|
|
|
chars = input("Enter characters: ") |
|
|
|
inputChars.send_keys(chars) |
|
|
|
|
|
|
|
num = input("Enter number of wrong puzzle pieces: ") |
|
|
|
inputNum.send_keys(num) |
|
|
|
|
|
|
|
# click the verify(submit) button |
|
|
|
driver.find_element(by=By.XPATH, value="/html/body/div[2]/div/div/div/div/form/div/div[2]/button").click() |
|
|
|
''' |
|
|
|
input("Press ENTER when CAPTCHA is completed\n") |
|
|
|
|
|
|
|
# wait for login page |
|
|
|
WebDriverWait(driver, 100).until(EC.visibility_of_element_located( |
|
|
|
(By.XPATH, "/html/body/div[2]/div/div/div[2]/h1"))) |
|
|
|
|
|
|
|
''' |
|
|
|
temp = driver.find_element(by=By.XPATH, value='/html/body/div/div/form/div[1]') |
|
|
|
boxes = temp.find_elements(by=By.TAG_NAME, value='input') |
|
|
|
|
|
|
|
for box in boxes: |
|
|
|
# click box to update captcha image |
|
|
|
box.click() |
|
|
|
|
|
|
|
# save clock captcha to local |
|
|
|
time.sleep(1) |
|
|
|
driver.find_element(by=By.XPATH, value='/html/body/div/div/form/div[1]/div').screenshot( |
|
|
|
r'..\Royal\captcha1.png') |
|
|
|
|
|
|
|
im = Image.open(r'..\Royal\captcha1.png') |
|
|
|
im.show() |
|
|
|
|
|
|
|
letter = input("Enter letter: ") |
|
|
|
box.send_keys(letter) |
|
|
|
|
|
|
|
# click the verify(submit) button |
|
|
|
driver.find_element(by=By.XPATH, value="/html/body/div/div/form/button[1]").click() |
|
|
|
|
|
|
|
# wait for login page |
|
|
|
WebDriverWait(driver, 100).until(EC.visibility_of_element_located( |
|
|
|
(By.XPATH, "/html/body/div[1]/div/div/div[2]/form/input[3]"))) |
|
|
|
''' |
|
|
|
|
|
|
|
|
|
|
|
# Login using premade account credentials and do login captcha manually |
|
|
|
def login(driver): |
|
|
|
# wait for login page |
|
|
|
WebDriverWait(driver, 100).until(EC.visibility_of_element_located( |
|
|
|
(By.XPATH, "/html/body/div[2]/div/div/div[2]/form/div[4]"))) |
|
|
|
|
|
|
|
# entering username and password into input boxes |
|
|
|
usernameBox = driver.find_element(by=By.XPATH, value='//*[@id="username"]') |
|
|
|
# Username here |
|
|
|
usernameBox.send_keys('blabri') |
|
|
|
passwordBox = driver.find_element(by=By.XPATH, value='//*[@id="password"]') |
|
|
|
# Password here |
|
|
|
passwordBox.send_keys('fishowal') |
|
|
|
|
|
|
|
# click "Login" |
|
|
|
driver.find_element(by=By.XPATH, value='/html/body/div[2]/div/div/div[2]/form/div[4]').click() |
|
|
|
|
|
|
|
''' |
|
|
|
# wait for captcha page show up |
|
|
|
time.sleep(3) |
|
|
|
|
|
|
|
# save captcha to local |
|
|
|
driver.find_element(by=By.XPATH, value='/html/body/div[2]/div/div/div[2]/form/div[4]/label/div/div').screenshot( |
|
|
|
r'..\Royal\captcha2.png') |
|
|
|
|
|
|
|
# This method will show image in any image viewer |
|
|
|
im = Image.open(r'..\Royal\captcha2.png') |
|
|
|
im.show() |
|
|
|
|
|
|
|
# ask user input captcha solution in terminal |
|
|
|
userIn = input("Enter location of wrong pieces (squares are numbered 1-24 left to right, # # #): ") |
|
|
|
squares = userIn.split() |
|
|
|
|
|
|
|
# send user solution into the input space |
|
|
|
for id in squares: |
|
|
|
driver.find_element(by=By.XPATH, value='//*[@id="cl[' + str((int(id)-1)) + ']"]').click() |
|
|
|
|
|
|
|
# click the verify(submit) button |
|
|
|
driver.find_element(by=By.XPATH, value="/html/body/div[2]/div/div/div[2]/form/div[4]/label/div/div/div/button").click() |
|
|
|
''' |
|
|
|
input("Press ENTER when CAPTCHA is completed\n") |
|
|
|
|
|
|
|
# wait for listing page show up (This Xpath may need to change based on different seed url) |
|
|
|
WebDriverWait(driver, 50).until(EC.visibility_of_element_located( |
|
|
|
(By.XPATH, '/html/body/div[3]/div/div[5]/div[1]'))) |
|
|
|
|
|
|
|
|
|
|
|
# Returns the name of the website |
|
|
|
def getMarketName(): |
|
|
|
name = 'TheDarkMarket' |
|
|
|
return name |
|
|
|
|
|
|
|
|
|
|
|
# Return the link of the website |
|
|
|
def getFixedURL(): |
|
|
|
url = 'http://dark3xolguutzr2cn5twjyu6c3db2z3ai3aqyqascml5cdrleh3s2hqd.onion/' |
|
|
|
|
|
|
|
return url |
|
|
|
|
|
|
|
|
|
|
|
# Closes Tor Browser |
|
|
|
def closeDriver(driver): |
|
|
|
# global pid |
|
|
|
# os.system("taskkill /pid " + str(pro.pid)) |
|
|
|
# os.system("taskkill /t /f /im tor.exe") |
|
|
|
print('Closing Tor...') |
|
|
|
driver.close() |
|
|
|
time.sleep(3) |
|
|
|
return |
|
|
|
|
|
|
|
|
|
|
|
# Creates FireFox 'driver' and configure its 'Profile' |
|
|
|
# to use Tor proxy and socket |
|
|
|
def createFFDriver(): |
|
|
|
from MarketPlaces.Initialization.markets_mining import config |
|
|
|
|
|
|
|
ff_binary = FirefoxBinary(config.get('TOR', 'firefox_binary_path')) |
|
|
|
|
|
|
|
ff_prof = FirefoxProfile(config.get('TOR', 'firefox_profile_path')) |
|
|
|
ff_prof.set_preference("places.history.enabled", False) |
|
|
|
ff_prof.set_preference("privacy.clearOnShutdown.offlineApps", True) |
|
|
|
ff_prof.set_preference("privacy.clearOnShutdown.passwords", True) |
|
|
|
ff_prof.set_preference("privacy.clearOnShutdown.siteSettings", True) |
|
|
|
ff_prof.set_preference("privacy.sanitize.sanitizeOnShutdown", True) |
|
|
|
ff_prof.set_preference("signon.rememberSignons", False) |
|
|
|
ff_prof.set_preference("network.cookie.lifetimePolicy", 2) |
|
|
|
# ff_prof.set_preference("network.dns.disablePrefetch", True) |
|
|
|
# ff_prof.set_preference("network.http.sendRefererHeader", 0) |
|
|
|
ff_prof.set_preference("permissions.default.image", 3) |
|
|
|
ff_prof.set_preference("browser.download.folderList", 2) |
|
|
|
ff_prof.set_preference("browser.download.manager.showWhenStarting", False) |
|
|
|
ff_prof.set_preference("browser.helperApps.neverAsk.saveToDisk", "text/plain") |
|
|
|
ff_prof.set_preference('network.proxy.type', 1) |
|
|
|
ff_prof.set_preference("network.proxy.socks_version", 5) |
|
|
|
ff_prof.set_preference('network.proxy.socks', '127.0.0.1') |
|
|
|
ff_prof.set_preference('network.proxy.socks_port', 9150) |
|
|
|
ff_prof.set_preference('network.proxy.socks_remote_dns', True) |
|
|
|
ff_prof.set_preference("javascript.enabled", False) |
|
|
|
ff_prof.update_preferences() |
|
|
|
|
|
|
|
service = Service(config.get('TOR', 'geckodriver_path')) |
|
|
|
|
|
|
|
driver = webdriver.Firefox(firefox_binary=ff_binary, firefox_profile=ff_prof, service=service) |
|
|
|
|
|
|
|
driver.maximize_window() |
|
|
|
|
|
|
|
return driver |
|
|
|
|
|
|
|
|
|
|
|
def getAccess(): |
|
|
|
url = getFixedURL() |
|
|
|
driver = createFFDriver() |
|
|
|
try: |
|
|
|
driver.get(url) |
|
|
|
return driver |
|
|
|
except: |
|
|
|
driver.close() |
|
|
|
return 'down' |
|
|
|
|
|
|
|
|
|
|
|
# Saves the crawled html page |
|
|
|
def savePage(driver, page, url): |
|
|
|
cleanPage = cleanHTML(driver, page) |
|
|
|
filePath = getFullPathName(url) |
|
|
|
os.makedirs(os.path.dirname(filePath), exist_ok=True) |
|
|
|
open(filePath, 'wb').write(cleanPage.encode('utf-8')) |
|
|
|
return |
|
|
|
|
|
|
|
|
|
|
|
# Gets the full path of the page to be saved along with its appropriate file name |
|
|
|
def getFullPathName(url): |
|
|
|
from MarketPlaces.Initialization.markets_mining import config, CURRENT_DATE |
|
|
|
|
|
|
|
mainDir = os.path.join(config.get('Project', 'shared_folder'), "MarketPlaces/" + getMarketName() + "/HTML_Pages") |
|
|
|
fileName = getNameFromURL(url) |
|
|
|
if not isListingLink(url): |
|
|
|
fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Description\\' + fileName + '.html') |
|
|
|
else: |
|
|
|
fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Listing\\' + fileName + '.html') |
|
|
|
return fullPath |
|
|
|
|
|
|
|
|
|
|
|
# Creates the file name from passed URL |
|
|
|
def getNameFromURL(url): |
|
|
|
global counter |
|
|
|
name = ''.join(e for e in url if e.isalnum()) |
|
|
|
if name == '': |
|
|
|
name = str(counter) |
|
|
|
counter = counter + 1 |
|
|
|
return name |
|
|
|
|
|
|
|
|
|
|
|
def getInterestedLinks(): |
|
|
|
links = [] |
|
|
|
|
|
|
|
# Digital - Fraud Software |
|
|
|
links.append(baseURL + 'product-category/hacking/') |
|
|
|
# # Digital - Guides and Tutorials |
|
|
|
# links.append('http://royalrnpvfbodtt5altnnzano6hquvn2d5qy55oofc2zyqciogcevrad.onion/category/Guides%20&%20Tutorials') |
|
|
|
# # Digital - Legitimate Software |
|
|
|
# links.append('http://royalrnpvfbodtt5altnnzano6hquvn2d5qy55oofc2zyqciogcevrad.onion/category/Legitimiate%20Software') |
|
|
|
# # Services - Carding |
|
|
|
# links.append('http://royalrnpvfbodtt5altnnzano6hquvn2d5qy55oofc2zyqciogcevrad.onion/category/Carding') |
|
|
|
|
|
|
|
return links |
|
|
|
|
|
|
|
|
|
|
|
def crawlForum(driver): |
|
|
|
print("Crawling The Dark Market") |
|
|
|
|
|
|
|
linksToCrawl = getInterestedLinks() |
|
|
|
|
|
|
|
i = 0 |
|
|
|
while i < len(linksToCrawl): |
|
|
|
link = linksToCrawl[i] |
|
|
|
print('Crawling :', link) |
|
|
|
try: |
|
|
|
has_next_page = True |
|
|
|
count = 0 |
|
|
|
|
|
|
|
while has_next_page: |
|
|
|
try: |
|
|
|
driver.get(link) |
|
|
|
except: |
|
|
|
driver.refresh() |
|
|
|
html = driver.page_source |
|
|
|
savePage(driver, html, link) |
|
|
|
|
|
|
|
list = productPages(html) |
|
|
|
for item in list: |
|
|
|
itemURL = urlparse.urljoin(baseURL, str(item)) |
|
|
|
try: |
|
|
|
driver.get(itemURL) |
|
|
|
except: |
|
|
|
driver.refresh() |
|
|
|
savePage(driver, driver.page_source, item) |
|
|
|
driver.back() |
|
|
|
|
|
|
|
# comment out |
|
|
|
# break |
|
|
|
|
|
|
|
# comment out |
|
|
|
# if count == 1: |
|
|
|
# break |
|
|
|
|
|
|
|
# Try finding next page |
|
|
|
try: |
|
|
|
nav = driver.find_element(by=By.XPATH, value='/html/body/div[2]/div/div/div[1]/div[2]/nav') |
|
|
|
li = nav.find_elements(by=By.TAG_NAME, value='li') |
|
|
|
a = li[-1].find_element(by=By.TAG_NAME, value='a') |
|
|
|
link = a.get_attribute('href') |
|
|
|
if link == "": |
|
|
|
raise NoSuchElementException |
|
|
|
count += 1 |
|
|
|
|
|
|
|
except NoSuchElementException: |
|
|
|
has_next_page = False |
|
|
|
|
|
|
|
except Exception as e: |
|
|
|
print(link, e) |
|
|
|
i += 1 |
|
|
|
|
|
|
|
input("Crawling Royal forum done sucessfully. Press ENTER to continue\n") |
|
|
|
|
|
|
|
|
|
|
|
# Returns 'True' if the link is Topic link |
|
|
|
def isDescriptionLink(url): |
|
|
|
if '/product/' in url: |
|
|
|
return True |
|
|
|
return False |
|
|
|
|
|
|
|
|
|
|
|
# Returns True if the link is a listingPage link |
|
|
|
def isListingLink(url): |
|
|
|
if 'category' in url: |
|
|
|
return True |
|
|
|
return False |
|
|
|
|
|
|
|
|
|
|
|
# calling the parser to define the links |
|
|
|
def productPages(html): |
|
|
|
soup = BeautifulSoup(html, "html.parser") |
|
|
|
return darkmarket_links_parser(soup) |
|
|
|
|
|
|
|
|
|
|
|
def crawler(): |
|
|
|
startCrawling() |
|
|
|
# print("Crawling and Parsing BestCardingWorld .... DONE!") |