You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
299 lines
9.6 KiB
299 lines
9.6 KiB
__author__ = 'DarkWeb'
|
|
|
|
'''
|
|
DWForums Forum Crawler (Selenium)
|
|
'''
|
|
|
|
from selenium import webdriver
|
|
from selenium.common.exceptions import NoSuchElementException
|
|
from selenium.webdriver.firefox.firefox_profile import FirefoxProfile
|
|
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
|
|
from selenium.webdriver.firefox.service import Service
|
|
from selenium.webdriver.common.by import By
|
|
from selenium.webdriver.support import expected_conditions as EC
|
|
from selenium.webdriver.support.ui import WebDriverWait
|
|
|
|
import urllib.parse as urlparse
|
|
import os, time
|
|
from datetime import date
|
|
import subprocess
|
|
from bs4 import BeautifulSoup
|
|
from Forums.Initialization.prepare_parser import new_parse
|
|
from Forums.DWForums.parser import dwForums_links_parser
|
|
from Forums.Utilities.utilities import cleanHTML
|
|
|
|
counter = 1
|
|
baseURL = 'http://dwforumuugiyderhybcpfxmlmoawgq6z3w6hk45nrnem3p7kwszhybad.onion/'
|
|
|
|
|
|
# Opens Tor Browser, crawls the website
|
|
def startCrawling():
|
|
# opentor()
|
|
# forumName = getForumName()
|
|
driver = getAccess()
|
|
|
|
if driver != 'down':
|
|
login(driver)
|
|
crawlForum(driver)
|
|
closetor(driver)
|
|
|
|
# new_parse(forumName, False)
|
|
|
|
|
|
# Opens Tor Browser
|
|
def opentor():
|
|
global pid
|
|
print("Connecting Tor...")
|
|
path = open('../../path.txt').readline().strip()
|
|
pro = subprocess.Popen(path)
|
|
pid = pro.pid
|
|
time.sleep(7.5)
|
|
input('Tor Connected. Press ENTER to continue\n')
|
|
return
|
|
|
|
# Login using premade account credentials and do login captcha manually
|
|
def login(driver):
|
|
#click login button
|
|
WebDriverWait(driver, 100).until(EC.visibility_of_element_located(
|
|
(By.CSS_SELECTOR, ".button--icon--user")))
|
|
login_link = driver.find_element_by_css_selector(".button--icon--user")
|
|
login_link.click()
|
|
|
|
#entering username and password into input boxes
|
|
WebDriverWait(driver, 100).until(EC.visibility_of_element_located(
|
|
(By.XPATH, "/html/body/div[4]/div/div[2]/div/form/div[1]")))
|
|
container = driver.find_element(by=By.XPATH, value="/html/body/div[4]/div/div[2]/div/form/div[1]")
|
|
# print(container.get_attribute("outerHTML"))
|
|
boxes = container.find_elements_by_class_name("input")
|
|
# print(len(boxes))
|
|
|
|
#Username here
|
|
boxes[0].send_keys('nice_reamer08')
|
|
#Password here
|
|
boxes[1].send_keys('tjpv$]Nc}XG@`%LM')
|
|
# no captcha on this site
|
|
|
|
# click the verify(submit) button
|
|
driver.find_element_by_css_selector(".button--icon--login").click()
|
|
|
|
# wait for listing page show up (This Xpath may need to change based on different seed url)
|
|
WebDriverWait(driver, 50).until(EC.visibility_of_element_located(
|
|
(By.CSS_SELECTOR, '.p-staffBar-inner > div:nth-child(4) > div:nth-child(1) > a:nth-child(1)')))
|
|
|
|
|
|
# Returns the name of the website
|
|
def getForumName():
|
|
name = 'DWForums'
|
|
return name
|
|
|
|
|
|
# Return the link of the website
|
|
def getFixedURL():
|
|
url = 'http://dwforumuugiyderhybcpfxmlmoawgq6z3w6hk45nrnem3p7kwszhybad.onion/'
|
|
return url
|
|
|
|
|
|
# Closes Tor Browser
|
|
def closetor(driver):
|
|
global pid
|
|
# os.system("taskkill /pid " + str(pro.pid))
|
|
os.system("taskkill /t /f /im tor.exe")
|
|
print('Closing Tor...')
|
|
driver.close()
|
|
time.sleep(3)
|
|
return
|
|
|
|
|
|
# Creates FireFox 'driver' and configure its 'Profile'
|
|
# to use Tor proxy and socket
|
|
def createFFDriver():
|
|
file = open('../../path.txt', 'r')
|
|
lines = file.readlines()
|
|
|
|
ff_binary = FirefoxBinary(lines[0].strip())
|
|
|
|
ff_prof = FirefoxProfile(lines[1].strip())
|
|
ff_prof.set_preference("places.history.enabled", False)
|
|
ff_prof.set_preference("privacy.clearOnShutdown.offlineApps", True)
|
|
ff_prof.set_preference("privacy.clearOnShutdown.passwords", True)
|
|
ff_prof.set_preference("privacy.clearOnShutdown.siteSettings", True)
|
|
ff_prof.set_preference("privacy.sanitize.sanitizeOnShutdown", True)
|
|
ff_prof.set_preference("signon.rememberSignons", False)
|
|
ff_prof.set_preference("network.cookie.lifetimePolicy", 2)
|
|
ff_prof.set_preference("network.dns.disablePrefetch", True)
|
|
ff_prof.set_preference("network.http.sendRefererHeader", 0)
|
|
ff_prof.set_preference("permissions.default.image", 3)
|
|
ff_prof.set_preference("browser.download.folderList", 2)
|
|
ff_prof.set_preference("browser.download.manager.showWhenStarting", False)
|
|
ff_prof.set_preference("browser.helperApps.neverAsk.saveToDisk", "text/plain")
|
|
ff_prof.set_preference('network.proxy.type', 1)
|
|
ff_prof.set_preference("network.proxy.socks_version", 5)
|
|
ff_prof.set_preference('network.proxy.socks', '127.0.0.1')
|
|
ff_prof.set_preference('network.proxy.socks_port', 9150)
|
|
ff_prof.set_preference('network.proxy.socks_remote_dns', True)
|
|
ff_prof.set_preference("javascript.enabled", True)
|
|
ff_prof.update_preferences()
|
|
|
|
service = Service(lines[2].strip())
|
|
|
|
driver = webdriver.Firefox(firefox_binary=ff_binary, firefox_profile=ff_prof, service=service)
|
|
|
|
return driver
|
|
|
|
|
|
def getAccess():
|
|
url = getFixedURL()
|
|
driver = createFFDriver()
|
|
|
|
try:
|
|
|
|
driver.get(url)
|
|
return driver
|
|
|
|
except:
|
|
|
|
return 'down'
|
|
|
|
|
|
# Saves the crawled html page
|
|
def savePage(page, url):
|
|
cleanPage = cleanHTML(page)
|
|
filePath = getFullPathName(url)
|
|
os.makedirs(os.path.dirname(filePath), exist_ok=True)
|
|
open(filePath, 'wb').write(cleanPage.encode('utf-8'))
|
|
return
|
|
|
|
|
|
# Gets the full path of the page to be saved along with its appropriate file name
|
|
def getFullPathName(url):
|
|
fileName = getNameFromURL(url)
|
|
if isDescriptionLink(url):
|
|
fullPath = r'..\DWForums\HTML_Pages\\' + str(
|
|
"%02d" % date.today().month) + str("%02d" % date.today().day) + str(
|
|
"%04d" % date.today().year) + r'\\' + r'Description\\' + fileName + '.html'
|
|
else:
|
|
fullPath = r'..\DWForums\HTML_Pages\\' + str(
|
|
"%02d" % date.today().month) + str("%02d" % date.today().day) + str(
|
|
"%04d" % date.today().year) + r'\\' + r'Listing\\' + fileName + '.html'
|
|
return fullPath
|
|
|
|
|
|
# Creates the file name from passed URL
|
|
def getNameFromURL(url):
|
|
global counter
|
|
name = ''.join(e for e in url if e.isalnum())
|
|
if (name == ''):
|
|
name = str(counter)
|
|
counter = counter + 1
|
|
return name
|
|
|
|
|
|
def getInterestedLinks():
|
|
links = []
|
|
|
|
# Hacking
|
|
links.append('http://dwforumuugiyderhybcpfxmlmoawgq6z3w6hk45nrnem3p7kwszhybad.onion/forums/hacking-forum.33/')
|
|
# # Beginner Carding and Fraud
|
|
links.append('http://dwforumuugiyderhybcpfxmlmoawgq6z3w6hk45nrnem3p7kwszhybad.onion/forums/remote-administration.34/')
|
|
# # Cracking Tools
|
|
links.append('http://dwforumuugiyderhybcpfxmlmoawgq6z3w6hk45nrnem3p7kwszhybad.onion/forums/cracking-tools.35/')
|
|
# # Cracking Tutorials and Other Methods - error here about file not exisitng
|
|
# links.append('http://dwforumuugiyderhybcpfxmlmoawgq6z3w6hk45nrnem3p7kwszhybad.onion/forums/cracking-tutorials-other-methods.36/')
|
|
# # Combolists and Configs
|
|
links.append('http://dwforumuugiyderhybcpfxmlmoawgq6z3w6hk45nrnem3p7kwszhybad.onion/forums/combolists-and-configs.58/')
|
|
# # Paid Software and Antivirus
|
|
links.append('http://dwforumuugiyderhybcpfxmlmoawgq6z3w6hk45nrnem3p7kwszhybad.onion/forums/paid-softwares-and-antivirus.59/')
|
|
|
|
return links
|
|
|
|
|
|
def crawlForum(driver):
|
|
print("Crawling the DWForums forum")
|
|
|
|
linksToCrawl = getInterestedLinks()
|
|
visited = set(linksToCrawl)
|
|
initialTime = time.time()
|
|
|
|
i = 0
|
|
count = 0
|
|
while i < len(linksToCrawl):
|
|
link = linksToCrawl[i]
|
|
print('Crawling :', link)
|
|
try:
|
|
try:
|
|
driver.get(link)
|
|
except:
|
|
driver.refresh()
|
|
html = driver.page_source
|
|
savePage(html, link)
|
|
|
|
has_next_page = True
|
|
while has_next_page:
|
|
list = topicPages(html)
|
|
for item in list:
|
|
itemURL = urlparse.urljoin(baseURL, str(item))
|
|
try:
|
|
driver.get(itemURL)
|
|
except:
|
|
driver.refresh()
|
|
savePage(driver.page_source, item)
|
|
driver.back()
|
|
# comment out
|
|
break
|
|
|
|
# comment out
|
|
if count == 1:
|
|
count = 0
|
|
break
|
|
|
|
try:
|
|
temp = driver.find_element_by_link_text("Next")
|
|
link = temp.get_attribute('href')
|
|
|
|
if link == "":
|
|
raise NoSuchElementException
|
|
try:
|
|
driver.get(link)
|
|
except:
|
|
driver.refresh()
|
|
html = driver.page_source
|
|
savePage(html, link)
|
|
count += 1
|
|
|
|
except NoSuchElementException:
|
|
has_next_page = False
|
|
|
|
except Exception as e:
|
|
print(link, e.message)
|
|
i += 1
|
|
|
|
# finalTime = time.time()
|
|
# print finalTime - initialTime
|
|
|
|
input("Crawling DWForums forum done sucessfully. Press ENTER to continue\n")
|
|
|
|
|
|
# Returns 'True' if the link is Topic link
|
|
def isDescriptionLink(url):
|
|
if '/threads/' in url:
|
|
return True
|
|
return False
|
|
|
|
|
|
# Returns True if the link is a listingPage link
|
|
def isListingLink(url):
|
|
if '/forums/' in url:
|
|
return True
|
|
return False
|
|
|
|
|
|
# calling the parser to define the links
|
|
def topicPages(html):
|
|
soup = BeautifulSoup(html, "html.parser")
|
|
#print(soup.find('div', id="container").find('div', id="content").find('table', {"class": "tborder clear"}).find('tbody').find('tr',{"class": "inline_row"}).find('strong').text)
|
|
return dwForums_links_parser(soup)
|
|
|
|
|
|
def crawler():
|
|
startCrawling()
|
|
# print("Crawling and Parsing BestCardingWorld .... DONE!")
|