__author__ = 'DarkWeb' ''' CryptBB Forum Crawler (Selenium) ''' from selenium import webdriver from selenium.common.exceptions import NoSuchElementException from selenium.webdriver.firefox.firefox_profile import FirefoxProfile from selenium.webdriver.firefox.firefox_binary import FirefoxBinary from selenium.webdriver.firefox.service import Service from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.ui import WebDriverWait from PIL import Image import urllib.parse as urlparse import os, re, time import subprocess from bs4 import BeautifulSoup from Forums.Initialization.prepare_parser import new_parse from Forums.CryptBB.parser import cryptBB_links_parser from Forums.Utilities.utilities import cleanHTML counter = 1 baseURL = 'http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/' # Opens Tor Browser, crawls the website def startCrawling(): forumName = getForumName() driver = getAccess() if driver != 'down': try: login(driver) crawlForum(driver) except Exception as e: print(driver.current_url, e) closeDriver(driver) new_parse(forumName, baseURL, True) # Login using premade account credentials and do login captcha manually def login(driver): #click login button login_link = driver.find_element( by=By.XPATH, value='/html/body/div/div[2]/div/table/tbody/tr[2]/td/center/pre/strong/a[1]').\ get_attribute('href') driver.get(login_link)# open tab with url #entering username and password into input boxes usernameBox = driver.find_element(by=By.XPATH, value='/html/body/div/div[2]/div/form/table/tbody/tr[2]/td[2]/input') #Username here usernameBox.send_keys('holyre')#sends string to the username box passwordBox = driver.find_element(by=By.XPATH, value='/html/body/div/div[2]/div/form/table/tbody/tr[3]/td[2]/input') #Password here passwordBox.send_keys('PlatinumBorn2')# sends string to passwordBox ''' # wait for captcha page show up WebDriverWait(driver, 100).until(EC.visibility_of_element_located( (By.XPATH, "/html/body/div/div[2]/div/form/div/input"))) # save captcha to local driver.find_element(by=By.XPATH, value='//*[@id="captcha_img"]').screenshot(r'..\CryptBB\captcha.png') # This method will show image in any image viewer im = Image.open(r'..\CryptBB\captcha.png') im.show() # wait until input space show up inputBox = driver.find_element(by=By.XPATH, value='//*[@id="imagestring"]') # ask user input captcha solution in terminal userIn = input("Enter solution: ") # send user solution into the input space inputBox.send_keys(userIn) # click the verify(submit) button driver.find_element(by=By.XPATH, value="/html/body/div/div[2]/div/form/div/input").click() ''' input("Press ENTER when CAPTCHA is completed\n") # wait for listing page show up (This Xpath may need to change based on different seed url) # wait for 50 sec until id = tab_content is found, then cont WebDriverWait(driver, 50).until(EC.visibility_of_element_located( (By.XPATH, '//*[@id="tab_content"]'))) # Returns the name of the website def getForumName() -> str: name = 'CryptBB' return name # Return the link of the website def getFixedURL(): url = 'http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/' return url # Closes Tor Browser def closeDriver(driver): # global pid # os.system("taskkill /pid " + str(pro.pid)) # os.system("taskkill /t /f /im tor.exe") print('Closing Tor...') driver.close() #close tab time.sleep(3) return # Creates FireFox 'driver' and configure its 'Profile' # to use Tor proxy and socket def createFFDriver(): from Forums.Initialization.forums_mining import config ff_binary = FirefoxBinary(config.get('TOR', 'firefox_binary_path')) ff_prof = FirefoxProfile(config.get('TOR', 'firefox_profile_path')) ff_prof.set_preference("places.history.enabled", False) ff_prof.set_preference("privacy.clearOnShutdown.offlineApps", True) ff_prof.set_preference("privacy.clearOnShutdown.passwords", True) ff_prof.set_preference("privacy.clearOnShutdown.siteSettings", True) ff_prof.set_preference("privacy.sanitize.sanitizeOnShutdown", True) ff_prof.set_preference("signon.rememberSignons", False) ff_prof.set_preference("network.cookie.lifetimePolicy", 2) ff_prof.set_preference("network.dns.disablePrefetch", True) ff_prof.set_preference("network.http.sendRefererHeader", 0) ff_prof.set_preference("permissions.default.image", 3) ff_prof.set_preference("browser.download.folderList", 2) ff_prof.set_preference("browser.download.manager.showWhenStarting", False) ff_prof.set_preference("browser.helperApps.neverAsk.saveToDisk", "text/plain") ff_prof.set_preference('network.proxy.type', 1) ff_prof.set_preference("network.proxy.socks_version", 5) ff_prof.set_preference('network.proxy.socks', '127.0.0.1') ff_prof.set_preference('network.proxy.socks_port', 9150) ff_prof.set_preference('network.proxy.socks_remote_dns', True) ff_prof.set_preference("javascript.enabled", True) ff_prof.update_preferences() service = Service(config.get('TOR', 'geckodriver_path')) driver = webdriver.Firefox(firefox_binary=ff_binary, firefox_profile=ff_prof, service=service) driver.maximize_window() return driver def getAccess(): url = getFixedURL() driver = createFFDriver() try: driver.get(url) return driver except: driver.close() return 'down' # Saves the crawled html page def savePage(driver, page, url): cleanPage = cleanHTML(driver, page) filePath = getFullPathName(url) os.makedirs(os.path.dirname(filePath), exist_ok=True) open(filePath, 'wb').write(cleanPage.encode('utf-8')) return # Gets the full path of the page to be saved along with its appropriate file name def getFullPathName(url): from Forums.Initialization.forums_mining import config, CURRENT_DATE mainDir = os.path.join(config.get('Project', 'shared_folder'), "Forums/" + getForumName() + "/HTML_Pages") fileName = getNameFromURL(url) if not isListingLink(url): fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Description\\' + fileName + '.html') else: fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Listing\\' + fileName + '.html') return fullPath # Creates the file name from passed URL def getNameFromURL(url): global counter name = ''.join(e for e in url if e.isalnum()) if name == '': name = str(counter) counter = counter + 1 return name def getInterestedLinks(): links = [] # Beginner Programming links.append('http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/forumdisplay.php?fid=86') # Beginner Carding and Fraud links.append('http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/forumdisplay.php?fid=91') # Beginner Hacking links.append('http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/forumdisplay.php?fid=87') # Newbie links.append('http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/forumdisplay.php?fid=84') # Beginner Hardware links.append('http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/forumdisplay.php?fid=89') # Training Challenges links.append('http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/forumdisplay.php?fid=96') # Darknet Discussions links.append('http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/forumdisplay.php?fid=88') # Public Leaks and Warez links.append('http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/forumdisplay.php?fid=97') # Sell links.append('http://cryptbbtg65gibadeeo2awe3j7s6evg7eklserehqr4w4e2bis5tebid.onion/forumdisplay.php?fid=44') return links def crawlForum(driver): print("Crawling the CryptBB forum") linksToCrawl = getInterestedLinks() i = 0 while i < len(linksToCrawl): link = linksToCrawl[i] print('Crawling :', link) try: has_next_page = True count = 0 while has_next_page: try: driver.get(link) except: driver.refresh() html = driver.page_source savePage(driver, html, link) topics = topicPages(html) for topic in topics: has_next_topic_page = True counter = 1 page = topic while has_next_topic_page: itemURL = urlparse.urljoin(baseURL, str(page)) try: driver.get(itemURL) except: driver.refresh() if isListingLink(driver.current_url): break savePage(driver, driver.page_source, topic + f"page{counter}") # very important # # comment out # if counter == 2: # break try: temp = driver.find_element(By.XPATH, '/html/body/div/div[2]/div/div[2]/div') page = temp.find_element(by=By.CLASS_NAME, value='pagination_next').get_attribute('href') if page == "": raise NoSuchElementException counter += 1 except NoSuchElementException: has_next_topic_page = False # making sure we go back to the listing page (browser back button simulation) try: driver.get(link) except: driver.refresh() # # comment out # break # # # comment out # if count == 1: # break try: temp = driver.find_element(by=By.XPATH, value='/html/body/div/div[2]/div/div[2]/div') link = temp.find_element(by=By.CLASS_NAME, value='pagination_next').get_attribute('href') if link == "": raise NoSuchElementException count += 1 except NoSuchElementException: has_next_page = False except Exception as e: print(link, e) i += 1 print("Crawling the CrypttBB forum done.") # Returns 'True' if the link is Topic link, may need to change for every website def isDescriptionLink(url): if 'thread' in url: return True return False # Returns True if the link is a listingPage link, may need to change for every website def isListingLink(url): if '.onion/forumdisplay' in url: return True return False # calling the parser to define the links def topicPages(html): soup = BeautifulSoup(html, "html.parser") return cryptBB_links_parser(soup) def crawler(): startCrawling() # print("Crawling and Parsing BestCardingWorld .... DONE!")