|
|
@ -29,14 +29,14 @@ baseURL = 'http://bestteermb42clir6ux7xm76d4jjodh3fpahjqgbddbmfrgp4skg2wqd.onion |
|
|
|
def startCrawling(): |
|
|
|
# opentor() |
|
|
|
forumName = getForumName() |
|
|
|
# driver = getAccess() |
|
|
|
driver = getAccess() |
|
|
|
|
|
|
|
# if driver != 'down': |
|
|
|
# try: |
|
|
|
# crawlForum(driver) |
|
|
|
# except Exception as e: |
|
|
|
# print(driver.current_url, e) |
|
|
|
# closetor(driver) |
|
|
|
if driver != 'down': |
|
|
|
try: |
|
|
|
crawlForum(driver) |
|
|
|
except Exception as e: |
|
|
|
print(driver.current_url, e) |
|
|
|
closetor(driver) |
|
|
|
|
|
|
|
new_parse(forumName, baseURL, True) |
|
|
|
|
|
|
@ -44,10 +44,11 @@ def startCrawling(): |
|
|
|
# Opens Tor Browser |
|
|
|
#prompts for ENTER input to continue |
|
|
|
def opentor(): |
|
|
|
from Forums.Initialization.forums_mining import config |
|
|
|
|
|
|
|
global pid |
|
|
|
print("Connecting Tor...") |
|
|
|
path = open('../../path.txt').readline().strip() |
|
|
|
pro = subprocess.Popen(path) |
|
|
|
pro = subprocess.Popen(config.get('TOR', 'firefox_binary_path')) |
|
|
|
pid = pro.pid |
|
|
|
time.sleep(7.5) |
|
|
|
input('Tor Connected. Press ENTER to continue\n') |
|
|
@ -71,9 +72,9 @@ def getFixedURL(): |
|
|
|
# Closes Tor Browser |
|
|
|
#@param: current selenium driver |
|
|
|
def closetor(driver): |
|
|
|
global pid |
|
|
|
# global pid |
|
|
|
# os.system("taskkill /pid " + str(pro.pid)) |
|
|
|
os.system("taskkill /t /f /im tor.exe") |
|
|
|
# os.system("taskkill /t /f /im tor.exe") |
|
|
|
print('Closing Tor...') |
|
|
|
driver.close() |
|
|
|
time.sleep(3) |
|
|
@ -83,12 +84,11 @@ def closetor(driver): |
|
|
|
# Creates FireFox 'driver' and configure its 'Profile' |
|
|
|
# to use Tor proxy and socket |
|
|
|
def createFFDriver(): |
|
|
|
file = open('../../path.txt', 'r') |
|
|
|
lines = file.readlines() |
|
|
|
from Forums.Initialization.forums_mining import config |
|
|
|
|
|
|
|
ff_binary = FirefoxBinary(lines[0].strip()) |
|
|
|
ff_binary = FirefoxBinary(config.get('TOR', 'firefox_binary_path')) |
|
|
|
|
|
|
|
ff_prof = FirefoxProfile(lines[1].strip()) |
|
|
|
ff_prof = FirefoxProfile(config.get('TOR', 'firefox_profile_path')) |
|
|
|
ff_prof.set_preference("places.history.enabled", False) |
|
|
|
ff_prof.set_preference("privacy.clearOnShutdown.offlineApps", True) |
|
|
|
ff_prof.set_preference("privacy.clearOnShutdown.passwords", True) |
|
|
@ -110,7 +110,7 @@ def createFFDriver(): |
|
|
|
ff_prof.set_preference("javascript.enabled", True) |
|
|
|
ff_prof.update_preferences() |
|
|
|
|
|
|
|
service = Service(lines[2].strip()) |
|
|
|
service = Service(config.get('TOR', 'geckodriver_path')) |
|
|
|
|
|
|
|
driver = webdriver.Firefox(firefox_binary=ff_binary, firefox_profile=ff_prof, service=service) |
|
|
|
|
|
|
@ -131,8 +131,8 @@ def getAccess(): |
|
|
|
|
|
|
|
|
|
|
|
# Saves the crawled html page, makes the directory path for html pages if not made |
|
|
|
def savePage(page, url): |
|
|
|
cleanPage = cleanHTML(page) |
|
|
|
def savePage(driver, page, url): |
|
|
|
cleanPage = cleanHTML(driver, page) |
|
|
|
filePath = getFullPathName(url) |
|
|
|
os.makedirs(os.path.dirname(filePath), exist_ok=True) |
|
|
|
open(filePath, 'wb').write(cleanPage.encode('utf-8')) |
|
|
@ -142,15 +142,14 @@ def savePage(page, url): |
|
|
|
# Gets the full path of the page to be saved along with its appropriate file name |
|
|
|
#@param: raw url as crawler crawls through every site |
|
|
|
def getFullPathName(url): |
|
|
|
from Forums.Initialization.forums_mining import config, CURRENT_DATE |
|
|
|
|
|
|
|
mainDir = os.path.join(config.get('Project', 'shared_folder'), "Forums/" + getForumName() + "/HTML_Pages") |
|
|
|
fileName = getNameFromURL(url) |
|
|
|
if isDescriptionLink(url): |
|
|
|
fullPath = r'C:\Users\fakeguy\Documents\threatIntelligence-main\DarkWebMining_Working\Forums\BestCardingWorld\HTML_Pages\\' + str( |
|
|
|
"%02d" % date.today().month) + str("%02d" % date.today().day) + str( |
|
|
|
"%04d" % date.today().year) + r'\\' + r'Description\\' + fileName + '.html' |
|
|
|
fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Description\\' + fileName + '.html') |
|
|
|
else: |
|
|
|
fullPath = r'C:\Users\fakeguy\Documents\threatIntelligence-main\DarkWebMining_Working\Forums\BestCardingWorld\HTML_Pages\\' + str( |
|
|
|
"%02d" % date.today().month) + str("%02d" % date.today().day) + str( |
|
|
|
"%04d" % date.today().year) + r'\\' + r'Listing\\' + fileName + '.html' |
|
|
|
fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Listing\\' + fileName + '.html') |
|
|
|
return fullPath |
|
|
|
|
|
|
|
|
|
|
@ -171,30 +170,26 @@ def getNameFromURL(url): |
|
|
|
def getInterestedLinks(): |
|
|
|
links = [] |
|
|
|
|
|
|
|
# Penetration Tests |
|
|
|
links.append('http://bestteermb42clir6ux7xm76d4jjodh3fpahjqgbddbmfrgp4skg2wqd.onion/viewforum.php?f=43') |
|
|
|
# # Penetration Tests |
|
|
|
# links.append('http://bestteermb42clir6ux7xm76d4jjodh3fpahjqgbddbmfrgp4skg2wqd.onion/viewforum.php?f=43') |
|
|
|
# # Social Engineering Tests |
|
|
|
links.append('http://bestteermb42clir6ux7xm76d4jjodh3fpahjqgbddbmfrgp4skg2wqd.onion/viewforum.php?f=44') |
|
|
|
# links.append('http://bestteermb42clir6ux7xm76d4jjodh3fpahjqgbddbmfrgp4skg2wqd.onion/viewforum.php?f=44') |
|
|
|
# # Exploits |
|
|
|
links.append('http://bestteermb42clir6ux7xm76d4jjodh3fpahjqgbddbmfrgp4skg2wqd.onion/viewforum.php?f=45') |
|
|
|
# links.append('http://bestteermb42clir6ux7xm76d4jjodh3fpahjqgbddbmfrgp4skg2wqd.onion/viewforum.php?f=45') |
|
|
|
# # Tools |
|
|
|
links.append('http://bestteermb42clir6ux7xm76d4jjodh3fpahjqgbddbmfrgp4skg2wqd.onion/viewforum.php?f=46') |
|
|
|
# # Malware |
|
|
|
# links.append('http://bestteermb42clir6ux7xm76d4jjodh3fpahjqgbddbmfrgp4skg2wqd.onion/viewforum.php?f=46') |
|
|
|
# Malware |
|
|
|
links.append('http://bestteermb42clir6ux7xm76d4jjodh3fpahjqgbddbmfrgp4skg2wqd.onion/viewforum.php?f=47') |
|
|
|
# # Cryptography |
|
|
|
links.append('http://bestteermb42clir6ux7xm76d4jjodh3fpahjqgbddbmfrgp4skg2wqd.onion/viewforum.php?f=48') |
|
|
|
# links.append('http://bestteermb42clir6ux7xm76d4jjodh3fpahjqgbddbmfrgp4skg2wqd.onion/viewforum.php?f=48') |
|
|
|
# # Others |
|
|
|
links.append('http://bestteermb42clir6ux7xm76d4jjodh3fpahjqgbddbmfrgp4skg2wqd.onion/viewforum.php?f=49') |
|
|
|
# links.append('http://bestteermb42clir6ux7xm76d4jjodh3fpahjqgbddbmfrgp4skg2wqd.onion/viewforum.php?f=49') |
|
|
|
# # Hacking Tutorials |
|
|
|
links.append('http://bestteermb42clir6ux7xm76d4jjodh3fpahjqgbddbmfrgp4skg2wqd.onion/viewforum.php?f=50') |
|
|
|
# links.append('http://bestteermb42clir6ux7xm76d4jjodh3fpahjqgbddbmfrgp4skg2wqd.onion/viewforum.php?f=50') |
|
|
|
# # Hacked Accounts and Database Dumps |
|
|
|
# links.append('http://bestteermb42clir6ux7xm76d4jjodh3fpahjqgbddbmfrgp4skg2wqd.onion/viewforum.php?f=30') |
|
|
|
# # Android Moded pak |
|
|
|
links.append('http://bestteermb42clir6ux7xm76d4jjodh3fpahjqgbddbmfrgp4skg2wqd.onion/viewforum.php?f=53') |
|
|
|
|
|
|
|
|
|
|
|
#General Discussion |
|
|
|
# links.append('http://bestteermb42clir6ux7xm76d4jjodh3fpahjqgbddbmfrgp4skg2wqd.onion/viewforum.php?f=16&sid=6a4959d49be41e72944e5aa5684c187a') |
|
|
|
# links.append('http://bestteermb42clir6ux7xm76d4jjodh3fpahjqgbddbmfrgp4skg2wqd.onion/viewforum.php?f=53') |
|
|
|
|
|
|
|
return links |
|
|
|
|
|
|
@ -206,45 +201,70 @@ def crawlForum(driver): |
|
|
|
print("Crawling the BestCardingWorld forum") |
|
|
|
|
|
|
|
linksToCrawl = getInterestedLinks() |
|
|
|
visited = set(linksToCrawl) |
|
|
|
initialTime = time.time() |
|
|
|
|
|
|
|
i = 0 |
|
|
|
while i < len(linksToCrawl): |
|
|
|
link = linksToCrawl[i] |
|
|
|
print('Crawling :', link) |
|
|
|
try: |
|
|
|
try: |
|
|
|
driver.get(link) |
|
|
|
except: |
|
|
|
driver.refresh() |
|
|
|
html = driver.page_source |
|
|
|
savePage(html, link) |
|
|
|
|
|
|
|
has_next_page = True |
|
|
|
count = 0 |
|
|
|
|
|
|
|
while has_next_page: |
|
|
|
list = topicPages(html) |
|
|
|
for item in list: |
|
|
|
itemURL = urlparse.urljoin(baseURL, str(item)) |
|
|
|
try: |
|
|
|
driver.get(itemURL) |
|
|
|
except: |
|
|
|
driver.refresh() |
|
|
|
savePage(driver.page_source, item) |
|
|
|
driver.back() |
|
|
|
try: |
|
|
|
driver.get(link) |
|
|
|
except: |
|
|
|
driver.refresh() |
|
|
|
html = driver.page_source |
|
|
|
savePage(driver, html, link) |
|
|
|
|
|
|
|
topics = topicPages(html) |
|
|
|
for topic in topics: |
|
|
|
has_next_topic_page = True |
|
|
|
counter = 1 |
|
|
|
page = topic |
|
|
|
|
|
|
|
while has_next_topic_page: |
|
|
|
itemURL = urlparse.urljoin(baseURL, str(page)) |
|
|
|
try: |
|
|
|
driver.get(itemURL) |
|
|
|
except: |
|
|
|
driver.refresh() |
|
|
|
savePage(driver, driver.page_source, topic + f"page{counter}") |
|
|
|
|
|
|
|
# comment out |
|
|
|
if counter == 2: |
|
|
|
break |
|
|
|
|
|
|
|
try: |
|
|
|
nav = driver.find_element(by=By.XPATH, value='/html/body/div[1]/div[2]/div[2]/div[4]/ul') |
|
|
|
li = nav.find_element_by_class_name('next') |
|
|
|
page = li.find_element_by_tag_name('a').get_attribute('href') |
|
|
|
if page == "": |
|
|
|
raise NoSuchElementException |
|
|
|
counter += 1 |
|
|
|
|
|
|
|
except NoSuchElementException: |
|
|
|
has_next_topic_page = False |
|
|
|
|
|
|
|
# end of loop |
|
|
|
for i in range(counter): |
|
|
|
driver.back() |
|
|
|
|
|
|
|
# comment out |
|
|
|
break |
|
|
|
|
|
|
|
# comment out |
|
|
|
if count == 1: |
|
|
|
break |
|
|
|
|
|
|
|
try: |
|
|
|
bar = driver.find_element(by=By.XPATH, value= |
|
|
|
'/html/body/div[1]/div[2]/div[2]/div[3]/ul') |
|
|
|
bar = driver.find_element(by=By.XPATH, value='/html/body/div[1]/div[2]/div[2]/div[3]/ul') |
|
|
|
next = bar.find_element_by_class_name('next') |
|
|
|
link = next.find_element_by_tag_name('a').get_attribute('href') |
|
|
|
|
|
|
|
try: |
|
|
|
driver.get(link) |
|
|
|
except: |
|
|
|
driver.refresh() |
|
|
|
html = driver.page_source |
|
|
|
savePage(html, link) |
|
|
|
if link == "": |
|
|
|
raise NoSuchElementException |
|
|
|
count += 1 |
|
|
|
|
|
|
|
except NoSuchElementException: |
|
|
|
has_next_page = False |
|
|
@ -253,9 +273,6 @@ def crawlForum(driver): |
|
|
|
print(link, e) |
|
|
|
i += 1 |
|
|
|
|
|
|
|
# finalTime = time.time() |
|
|
|
# print finalTime - initialTime |
|
|
|
|
|
|
|
input("Crawling BestCardingWorld forum done sucessfully. Press ENTER to continue\n") |
|
|
|
|
|
|
|
|
|
|
|