From 1a2d29d94fe4b9bb58ee9751ca4646b83132ba54 Mon Sep 17 00:00:00 2001 From: Joshua Date: Mon, 13 Nov 2023 08:47:20 -0800 Subject: [PATCH 1/5] setup configs for gofish crawling and parsing --- MarketPlaces/GoFish/crawler_selenium.py | 92 ++-- MarketPlaces/GoFish/parser.py | 531 +++++++++++------------- 2 files changed, 284 insertions(+), 339 deletions(-) diff --git a/MarketPlaces/GoFish/crawler_selenium.py b/MarketPlaces/GoFish/crawler_selenium.py index e5af35b..dbdbd50 100644 --- a/MarketPlaces/GoFish/crawler_selenium.py +++ b/MarketPlaces/GoFish/crawler_selenium.py @@ -1,8 +1,7 @@ __author__ = 'DarkWeb' ''' -Go Fish market Crawler (Selenium) -- this is a new marketplace and was up for only a few days, crawler has not been finished +GoFish Marketplace Crawler (Selenium) ''' from selenium import webdriver @@ -11,6 +10,7 @@ from selenium.webdriver.firefox.firefox_profile import FirefoxProfile from selenium.webdriver.firefox.firefox_binary import FirefoxBinary from selenium.webdriver.firefox.service import Service from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support.ui import Select from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.common.by import By @@ -28,8 +28,7 @@ from MarketPlaces.Utilities.utilities import cleanHTML counter = 1 baseURL = 'http://gofishbybookb4a2kvviuygmwjqfxx7nqsovweogs2cxvqvexhe7edyd.onion/' -# Opens Tor Browser, crawls the website, then parses, then closes tor -#acts like the main method for the crawler, another function at the end of this code calls this function later + def startCrawling(): mktName = getMKTName() driver = getAccess() @@ -46,21 +45,18 @@ def startCrawling(): # Returns the name of the website -#return: name of site in string type def getMKTName(): name = 'GoFish' return name # Return the base link of the website -#return: url of base site in string type def getFixedURL(): url = 'http://gofishbybookb4a2kvviuygmwjqfxx7nqsovweogs2cxvqvexhe7edyd.onion/' return url # Closes Tor Browser -#@param: current selenium driver def closeDriver(driver): # global pid # os.system("taskkill /pid " + str(pro.pid)) @@ -110,7 +106,6 @@ def createFFDriver(): #the driver 'gets' the url, attempting to get on the site, if it can't access return 'down' -#return: return the selenium driver or string 'down' def getAccess(): url = getFixedURL() driver = createFFDriver() @@ -122,31 +117,27 @@ def getAccess(): return 'down' -# Manual captcha solver, waits fora specific element so that the whole page loads, finds the input box, gets screenshot of captcha -# then allows for manual solving of captcha in the terminal -#@param: current selenium web driver def login(driver): - - # wait for page to show up (This Xpath may need to change based on different seed url) - WebDriverWait(driver, 100).until(EC.visibility_of_element_located( - (By.XPATH, '//*[@id="username"]'))) + input("Press ENTER when CAPTCHA is complete and login page has loaded\n") # entering username and password into input boxes - usernameBox = driver.find_element(by=By.XPATH, value='//*[@id="username"]') + usernameBox = driver.find_element(by=By.XPATH, value='//input[@name="username"]') # Username here - usernameBox.send_keys('itsmedio') - passwordBox = driver.find_element(by=By.XPATH, value='//*[@id="password"]') + usernameBox.send_keys('aliciamykeys') + passwordBox = driver.find_element(by=By.XPATH, value='//input[@name="password"]') # Password here - passwordBox.send_keys('DementedBed123-') + passwordBox.send_keys('aliciawherearemykey$') + # session time + session_select = Select(driver.find_element(by=By.XPATH, value='/html/body/main/div/div/div/div/div/form/div[4]/div/div[2]/select')) + session_select.select_by_visible_text('Session 60min') - input("Press ENTER when CAPTCHA is completed and logged in\n") + input("Press ENTER when CAPTCHA is completed and you exit the newsletter\n") # wait for listing page show up (This Xpath may need to change based on different seed url) WebDriverWait(driver, 100).until(EC.visibility_of_element_located( - (By.XPATH, "/html/body/div[1]/div[3]/div[1]/div[3]/ul/div"))) + (By.XPATH, '//*[@id="submit"]'))) -# Saves the crawled html page, makes the directory path for html pages if not made def savePage(driver, page, url): cleanPage = cleanHTML(driver, page) filePath = getFullPathName(url) @@ -155,8 +146,6 @@ def savePage(driver, page, url): return -# Gets the full path of the page to be saved along with its appropriate file name -#@param: raw url as crawler crawls through every site def getFullPathName(url): from MarketPlaces.Initialization.markets_mining import config, CURRENT_DATE @@ -169,38 +158,41 @@ def getFullPathName(url): return fullPath -# Creates the file name from passed URL, gives distinct name if can't be made unique after cleaned -#@param: raw url as crawler crawls through every site +def getMKTName() -> str: + name = 'GoFish' + return name + + def getNameFromURL(url): global counter name = ''.join(e for e in url if e.isalnum()) - if (name == ''): + if name == '': name = str(counter) counter = counter + 1 return name -# returns list of urls, here is where you can list the different urls of interest, the crawler runs through this list -#in this example, there are a couple of categories some threads fall under such as -# Guides and Tutorials, Digital Products, and Software and Malware -#as you can see they are categories of products def getInterestedLinks(): links = [] - # Hacking - links.append('http://gofishbybookb4a2kvviuygmwjqfxx7nqsovweogs2cxvqvexhe7edyd.onion/?c=129') - # Malware - links.append('http://gofishbybookb4a2kvviuygmwjqfxx7nqsovweogs2cxvqvexhe7edyd.onion/?c=97') - # Exploits + # Hosting and Security + links.append('http://gofishbybookb4a2kvviuygmwjqfxx7nqsovweogs2cxvqvexhe7edyd.onion/?c=84') + # Exploits and Kits links.append('http://gofishbybookb4a2kvviuygmwjqfxx7nqsovweogs2cxvqvexhe7edyd.onion/?c=107') + # Botnets and Malware + links.append('http://gofishbybookb4a2kvviuygmwjqfxx7nqsovweogs2cxvqvexhe7edyd.onion/?c=97') + # Other Software + links.append('http://gofishbybookb4a2kvviuygmwjqfxx7nqsovweogs2cxvqvexhe7edyd.onion/?c=108') + # Hacking Guide + links.append('http://gofishbybookb4a2kvviuygmwjqfxx7nqsovweogs2cxvqvexhe7edyd.onion/?c=129') + # Fraud (mostly carding) + links.append('http://gofishbybookb4a2kvviuygmwjqfxx7nqsovweogs2cxvqvexhe7edyd.onion/?c=128') return links -# gets links of interest to crawl through, iterates through list, where each link is clicked and crawled through -#topic and description pages are crawled through here, where both types of pages are saved -#@param: selenium driver def crawlForum(driver): + print("Crawling the GoFish market") linksToCrawl = getInterestedLinks() @@ -222,6 +214,7 @@ def crawlForum(driver): savePage(driver, html, link) list = productPages(html) + for item in list: itemURL = urlparse.urljoin(baseURL, str(item)) try: @@ -229,7 +222,6 @@ def crawlForum(driver): except: driver.refresh() savePage(driver, driver.page_source, item) - time.sleep(3) driver.back() # comment out @@ -240,7 +232,7 @@ def crawlForum(driver): break try: - link = driver.find_element(by=By.LINK_TEXT, value='Next ›').get_attribute('href') + link = driver.find_element(by=By.XPATH, value='//a[contains(text(), "Next")]').get_attribute('href') if link == "": raise NoSuchElementException count += 1 @@ -252,30 +244,23 @@ def crawlForum(driver): print(link, e) i += 1 - print("Crawling the GoFishMarket done.") + print("Crawling the GoFish market done.") -# Returns 'True' if the link is a description link -#@param: url of any url crawled -#return: true if is a description page, false if not +# Returns 'True' if the link is Topic link, may need to change for every website def isDescriptionLink(url): - if 'c' in url: + if 'item' in url: return True return False -# Returns True if the link is a listingPage link -#@param: url of any url crawled -#return: true if is a Listing page, false if not +# Returns True if the link is a listingPage link, may need to change for every website def isListingLink(url): - if 'a' in url: + if 'category=' in url: return True return False -# calling the parser to define the links, the html is the url of a link from the list of interested link list -#@param: link from interested link list ie. getInterestingLinks() -#return: list of description links that should be crawled through def productPages(html): soup = BeautifulSoup(html, "html.parser") return gofish_links_parser(soup) @@ -283,4 +268,3 @@ def productPages(html): def crawler(): startCrawling() - # print("Crawling and Parsing BestCardingWorld .... DONE!") diff --git a/MarketPlaces/GoFish/parser.py b/MarketPlaces/GoFish/parser.py index 28e1a6b..61cbd48 100644 --- a/MarketPlaces/GoFish/parser.py +++ b/MarketPlaces/GoFish/parser.py @@ -7,321 +7,282 @@ from MarketPlaces.Utilities.utilities import * from bs4 import BeautifulSoup -#parses description pages, so takes html pages of description pages using soup object, and parses it for info it needs -#stores info it needs in different lists, these lists are returned after being organized -#@param: soup object looking at html page of description page -#return: 'row' that contains a variety of lists that each hold info on the description page +# parses description pages, so takes html pages of description pages using soup object, and parses it for info it needs +# stores info it needs in different lists, these lists are returned after being organized +# @param: soup object looking at html page of description page +# return: 'row' that contains a variety of lists that each hold info on the description page def gofish_description_parser(soup): - # Fields to be parsed - vendor = "-1" # 0 *Vendor_Name - success = "-1" # 1 Vendor_Successful_Transactions - rating_vendor = "-1" # 2 Vendor_Rating - name = "-1" # 3 *Product_Name - describe = "-1" # 4 Product_Description - CVE = "-1" # 5 Product_CVE_Classification (Common Vulnerabilities and Exposures) dont worry about that much - MS = "-1" # 6 Product_MS_Classification (Microsoft Security) dont worry about that much - category = "-1" # 7 Product_Category - views = "-1" # 8 Product_Number_Of_Views - reviews = "-1" # 9 Product_Number_Of_Reviews - rating_item = "-1" # 10 Product_Rating - addDate = "-1" # 11 Product_AddedDate - BTC = "-1" # 12 Product_BTC_SellingPrice - USD = "-1" # 13 Product_USD_SellingPrice - EURO = "-1" # 14 Product_EURO_SellingPrice - sold = "-1" # 15 Product_QuantitySold - left = "-1" # 16 Product_QuantityLeft - shipFrom = "-1" # 17 Product_ShippedFrom - shipTo = "-1" # 18 Product_ShippedTo - - #vendor name - try: - temp = soup.find('div', {'class': 'box rounded mb-0'}).find('a').text - vendor = (cleanString(temp.strip())) - except: - vendor = "-1" - - #successful transaction - try: - temp = soup.findAll('div', {'class','text-center text-truncate column-flex ml-1 mr-1'}) #card sidebar-menu mb-4 card sidebar-menu mb-4 - temp2 = temp[1].findAll('span', {'class', 'float-right font-weight-bold'}) - temp = temp2[1].text - success = (temp.strip()) - except: - print("success") - - - #vendor rating 5 - try: - temp = soup.findAll('div', {'class', 'text-center text-truncate column-flex ml-1 mr-1'}) # card sidebar-menu mb-4 card sidebar-menu mb-4 - temp2 = temp[1].findAll('span', {'class', 'float-right font-weight-bold'}) - temp = temp2[5].text - rating_vendor = (cleanString(temp.strip())) - except: - print("product") - - # product name - try: - temp = soup.find('h3', {'class', 'h3 rounded card-title'}).find('span').text - name = (cleanString(temp.strip())) - except: - temp = soup.find('h3', {'class', 'h3 rounded card-title'}).find('span').find("div").text - name = (cleanString(temp.strip())) - - # product description - describe = soup.find('div', {'class': "box rounded flex-fill"}).find('pre').text - if "\n" in describe: - describe = describe.replace("\n", " ") - describe = describe.replace("\r", " ") - describe = cleanString(describe.strip()) - - CVE = "-1" # 5 Product_CVE_Classification (Common Vulnerabilities and Exposures) dont worry about that much - MS = "-1" # 6 Product_MS_Classification (Microsoft Security) dont worry about that much - - # product category - try: - temp = soup.findAll('table', {'class', 'table table-hover'}) - temp2 = temp[1].find('tr').findAll('td') - temp = temp2[1].text - category = cleanString(temp.strip()) - except: - temp = soup.find('table', {'class', 'table table-hover'}) - temp2 = temp.find('tbody').find('tr').findAll('td') - temp = temp2[1].text - category = cleanString(temp.strip()) - - # product number of view - try: - temp = soup.find('div', {'class', 'box rounded mb-0'}) - temp2 = temp.findAll('i') - temp = temp2[2].text - views = cleanString((temp.strip())) - except: - print('Product number of view') - - reviews = "-1" # 9 Product_Number_Of_Reviews - rating_item = "-1" # 10 Product_Rating - addDate = "-1" # 11 Product_AddedDate - - #BTC selling price box box-rounded mt-2 - try: - temp = soup.find('div', {'class', 'box box-rounded mt-2'}) - temp2 = temp.findAll('i', {'class', 'float-right color-prices'}) - temp = temp2[1].text - BTC = cleanString((temp.strip())) - except: - try: - temp = soup.find('div', {'class', 'box box-rounded'}) - temp2 = temp.findAll('span', {'class', 'float-right color-prices'}) - temp = temp2[1].text - BTC = cleanString((temp.strip())) - except: - print("BTC") - - - # USD selling price - try: - temp = soup.find('div', {'class', 'box box-rounded mt-2'}) - temp2 = temp.findAll('center') - temp = temp2[1].find('i').text - if "$" in temp: - temp = temp.replace("$", "") - USD = cleanString((temp.strip())) - except: - try: - temp = soup.find('div', {'class', 'box box-rounded'}) - temp2 = temp.findAll('center') - temp = temp2[1].find('span').text - if "$" in temp: - temp = temp.replace("$", "") - USD = cleanString((temp.strip())) - except: - print("USD") - - EURO = "-1" # 14 Product_EURO_SellingPrice - - - # product sold - try: - temp = soup.find('div', {'class', 'box rounded mb-0'}) # card sidebar-menu mb-4 card sidebar-menu mb-4 - temp2 = temp.find('i') - temp = temp2.text - sold = (cleanString(temp.strip())) - # sold = "-1" - except: - print("product sold") - - # product quantatiy left ###ERRROR - try: - temp = soup.findAll('table', {'class', 'table table-hover'}) - temp2 = temp[1].findAll('tr') - temp3 = temp2[1].findAll('td') - temp = temp3[1].text - left = cleanString(temp.strip()) - except: - temp = soup.find('table', {'class', 'table table-hover'}) - temp2 = temp.findAll('tr') - temp3 = temp2[1].findAll('td') - temp = temp3[1].text - left = cleanString(temp.strip()) - - - shipFrom = "-1" # 17 Product_ShippedFrom - shipTo = "-1" # 18 Product_ShippedTo + + vendor = "-1" # 0 *Vendor_Name + success = "-1" # 1 Vendor_Successful_Transactions + rating_vendor = "-1" # 2 Vendor_Rating + name = "-1" # 3 *Product_Name + describe = "-1" # 4 Product_Description + CVE = "-1" # 5 Product_CVE_Classification (Common Vulnerabilities and Exposures) + MS = "-1" # 6 Product_MS_Classification (Microsoft Security) + category = "-1" # 7 Product_Category + views = "-1" # 8 Product_Number_Of_Views + reviews = "-1" # 9 Product_Number_Of_Reviews + rating_item = "-1" # 10 Product_Rating + addDate = "-1" # 11 Product_AddedDate + BTC = "-1" # 12 Product_BTC_SellingPrice + USD = "-1" # 13 Product_USD_SellingPrice + EURO = "-1" # 14 Product_EURO_SellingPrice + sold = "-1" # 15 Product_QuantitySold + left = "-1" # 16 Product_QuantityLeft + shipFrom = "-1" # 17 Product_ShippedFrom + shipTo = "-1" # 18 Product_ShippedTo + image = "-1" # 19 Product_Image + vendor_image = "-1" # 20 Vendor_Image + + # Finding Product Name + divmb = soup.findAll('div', {'class': "mb-1"}) + + name = divmb[0].text + name = name.replace('\n', ' ') + name = name.replace(",", "") + name = name.strip() + + # Finding Vendor + vendor = divmb[1].find('a').text.strip() + + # Finding Vendor Rating + temp = soup.find('div', {'class': ""}).text + temp = temp.split('(') + rating = temp[0].replace("Vendor's Review : ", "") + rating = rating.replace("%", "") + rating_vendor = rating.strip() + + # Finding the Product Rating and Number of Product Reviews + reviews = temp[2].replace(" review)", "") + reviews = reviews.strip() + + temp = temp[1].split(")") + rating = temp[1].replace("Product Review : ", "") + rating = rating.replace("%", "") + rating_item = rating.strip() + + # Finding Prices + USD = soup.find('div', {'class': "h3 text-primary"}).text.strip() + + # Finding the Product Category + pmb = soup.findAll('p', {'class': "mb-1"}) + + category = pmb[-1].text + category = category.replace("Category: ", "").strip() + + # Finding the Product Quantity Available + left = divmb[-1].text + left = left.split(",", 1)[1] + left = left.replace("in stock", "") + left = left.strip() + + # Finding Number Sold + sold = divmb[-1].text + sold = sold.split(",", 1)[0] + sold = sold.replace("sold", "") + sold = sold.strip() + + # Finding Shipment Information (Origin) + pmb[0].text + shipFrom = shipFrom.replace("Ships from: ", "").strip() + + # Finding Shipment Information (Destination) + pmb[1].text + shipTo = shipTo.replace("Ships to: ", "").strip() + + # Finding the Product description + cardbody = soup.findAll('div', {'class': "card-body"}) + describe = cardbody[1].text.strip() + + # Finding Product Image + image = soup.find('div', {'class': 'product-primary'}).find('img') + image = image.get('src') + image = image.split('base64,')[-1] + + # Searching for CVE and MS categories + cve = soup.findAll(text=re.compile('CVE-\d{4}-\d{4}')) + if cve: + CVE = " " + for idx in cve: + CVE += (idx) + CVE += " " + CVE = CVE.replace(',', ' ') + CVE = CVE.replace('\n', '') + ms = soup.findAll(text=re.compile('MS\d{2}-\d{3}')) + if ms: + MS = " " + for im in ms: + MS += (im) + MS += " " + MS = MS.replace(',', ' ') + MS = MS.replace('\n', '') # Populating the final variable (this should be a list with all fields scraped) row = (vendor, rating_vendor, success, name, describe, CVE, MS, category, views, reviews, rating_item, addDate, - BTC, USD, EURO, sold, left, shipFrom, shipTo) + BTC, USD, EURO, sold, left, shipFrom, shipTo, image, vendor_image) # Sending the results return row -#parses listing pages, so takes html pages of listing pages using soup object, and parses it for info it needs -#stores info it needs in different lists, these lists are returned after being organized -#@param: soup object looking at html page of listing page -#return: 'row' that contains a variety of lists that each hold info on the listing page +# parses listing pages, so takes html pages of listing pages using soup object, and parses it for info it needs +# stores info it needs in different lists, these lists are returned after being organized +# @param: soup object looking at html page of listing page +# return: 'row' that contains a variety of lists that each hold info on the listing page def gofish_listing_parser(soup): + # Fields to be parsed nm = 0 # *Total_Products (Should be Integer) - mktName = "GoFish" # 0 *Marketplace_Name - vendor = [] # 1 *Vendor y - rating_vendor = [] # 2 Vendor_Rating - success = [] # 3 Vendor_Successful_Transactions - name = [] # 4 *Product_Name y - CVE = [] # 5 Product_CVE_Classification (Common Vulnerabilities and Exposures) dont worry about this - MS = [] # 6 Product_MS_Classification (Microsoft Security) dont worry about this - category = [] # 7 Product_Category y - describe = [] # 8 Product_Description - views = [] # 9 Product_Number_Of_Views - reviews = [] # 10 Product_Number_Of_Reviews - rating_item = [] # 11 Product_Rating - addDate = [] # 12 Product_AddDate - BTC = [] # 13 Product_BTC_SellingPrice - USD = [] # 14 Product_USD_SellingPrice y - EURO = [] # 15 Product_EURO_SellingPrice - sold = [] # 16 Product_QuantitySold - qLeft = [] # 17 Product_QuantityLeft - shipFrom = [] # 18 Product_ShippedFrom - shipTo = [] # 19 Product_ShippedTo - href = [] # 20 Product_Links - - listing = soup.findAll('div', {"class": "card mt-1"}) + mktName = "GoFish" # 0 *Marketplace_Name + vendor = [] # 1 *Vendor y + rating_vendor = [] # 2 Vendor_Rating + success = [] # 3 Vendor_Successful_Transactions + name = [] # 4 *Product_Name y + CVE = [] # 5 Product_CVE_Classification (Common Vulnerabilities and Exposures) dont worry about this + MS = [] # 6 Product_MS_Classification (Microsoft Security) dont worry about this + category = [] # 7 Product_Category y + describe = [] # 8 Product_Description + views = [] # 9 Product_Number_Of_Views + reviews = [] # 10 Product_Number_Of_Reviews + rating_item = [] # 11 Product_Rating + addDate = [] # 12 Product_AddDate + BTC = [] # 13 Product_BTC_SellingPrice + USD = [] # 14 Product_USD_SellingPrice y + EURO = [] # 15 Product_EURO_SellingPrice + sold = [] # 16 Product_QuantitySold + qLeft = [] # 17 Product_QuantityLeft + shipFrom = [] # 18 Product_ShippedFrom + shipTo = [] # 19 Product_ShippedTo + image = [] # 20 Product_Image + image_vendor = [] # 21 Vendor_Image + href = [] # 22 Product_Links + + listing = soup.findAll('div', {"id": "itembox"}) # Populating the Number of Products nm = len(listing) for a in listing: + bae = a.findAll('a', href=True) + lb = a.findAll('div', {"id": "littlebox"}) - # vendor - try: - temp = a.find('div', {'class','col-5 justify-content-between mx-auto'}).find('a').text - vendor.append(cleanString(temp.strip())) - except: - print('vendor') - - #vendor rating - - - #successful transactions CHECK AGAIN HERE - try: - success.append("-1") - except: - print('successful transactions') - - # product name - try: - temp = a.find('h5', {'class','card-title rounded text-truncate'}).find('a').text - name.append(cleanString(temp.strip())) - except: - print('product name') - - - CVE.append('-1') - MS.append('-1') - rating_vendor.append('-1') - - # product category - try: - temp = soup.find('div', {'class', 'card-sidebar-menu box mb-2 flex-column'}).find('h3').find('span').text - if "Search Results for: " in temp: - temp = temp.replace("Search Results for: ", "") - category.append(cleanString(temp.strip())) - - except: - print("Error in product category") - - describe.append('-1') - - # product views - try: - temp = a.find('h6',{'class', 'card-subtitle mb-1 text-muted text-truncate'}) - temp2 = temp.find('i').text - views.append(cleanString(temp2.strip())) - except: - print("Error in views") - - reviews.append('-1') # 10 Product_Number_Of_Reviews - rating_item.append('-1') # 11 Product_Rating - addDate.append('-1') # 12 Product_AddDate - - # BTC - try: - temp = a.find('div', {'class', 'col-3 justify-content-between mx-auto'}) - temp2 = temp.findAll('p') - temp = temp2[1].text - BTC.append(cleanString(temp.strip())) - except: - print("BTC") - - #USD ERROR get rid of $ - try: - temp = a.find('div', {'class', 'col-12 justify-content-between mx-auto'}).find('i').text - if '$' in temp: - temp = temp.replace("$", "") - USD.append(cleanString(temp.strip())) # 14 Product_USD_SellingPrice - except: - print("USD") - - EURO.append("-1") # 15 Product_EURO_SellingPrice - - #product sold - try: - temp = a.find('div', {'class', 'col-12 mx-auto text-truncate text-center flex-fill'}).findAll('p', {'class', 'card-text mb-0'}) - temp2 = temp[1].find('i').text - sold.append(cleanString(temp2.strip())) - except: - print("product sold") - - qLeft.append('-1') # 17 Product_QuantityLeft - shipFrom.append('-1') # 18 Product_ShippedFrom - shipTo.append('-1') # 19 Product_ShippedTo - - #href - try: - temp = a.find('h5', {'class', 'card-title rounded text-truncate'}).find('a').get('href') - href.append(temp) # 20 Product_Links - except: - print("href") + # Adding the url to the list of urls + link = bae[0].get('href') + href.append(link) + # Finding the Product + product = lb[1].find('a').text + product = product.replace('\n', ' ') + product = product.replace(",", "") + product = product.replace("...", "") + product = product.strip() + name.append(product) + + # Finding Product Image + product_image = a.find('img') + product_image = product_image.get('src') + product_image = product_image.split('base64,')[-1] + image.append(product_image) + + # Finding Prices + price = lb[-1].find('div', {"class": "mb-1"}).text + price = price.replace("$","") + price = price.strip() + USD.append(price) + + # Finding the Vendor + vendor_name = lb[-1].find("a").text + vendor_name = vendor_name.replace(",", "") + vendor_name = vendor_name.strip() + vendor.append(vendor_name) + + image_vendor.append("-1") + + # Finding the Category + cat = lb[-1].find("span").text + cat = cat.replace("class:", "") + cat = cat.strip() + category.append(cat) + + span = lb[0].findAll("span") + + # Finding Number of Views + num = span[0].text + num = num.replace("views:", "") + num = num.strip() + sold.append(num) + + # Finding Number Sold + num = span[2].text + num = num.replace("Sold:", "") + num = num.strip() + sold.append(num) + + # Finding Quantity Left + quant = span[1].text + quant = quant.replace("stock:", "") + quant = quant.strip() + qLeft.append(quant) + + # add shipping information + ship = lb[2].findAll('small')[1].findAll('span')[1].text.split("->") + shipFrom.append(ship[0].replace("Ship from ", "").strip()) + shipTo.append(ship[1].replace("to ", "").strip()) + + + # Searching for CVE and MS categories + cve = a.findAll(text=re.compile('CVE-\d{4}-\d{4}')) + if not cve: + cveValue = "-1" + else: + cee = " " + for idx in cve: + cee += (idx) + cee += " " + cee = cee.replace(',', ' ') + cee = cee.replace('\n', '') + cveValue = cee + CVE.append(cveValue) + + ms = a.findAll(text=re.compile('MS\d{2}-\d{3}')) + if not ms: + MSValue = "-1" + else: + me = " " + for im in ms: + me += (im) + me += " " + me = me.replace(',', ' ') + me = me.replace('\n', '') + MSValue = me + MS.append(MSValue) # Populate the final variable (this should be a list with all fields scraped) return organizeProducts(mktName, nm, vendor, rating_vendor, success, name, CVE, MS, category, describe, views, - reviews, rating_item, addDate, BTC, USD, EURO, sold, qLeft, shipFrom, shipTo, href) + reviews, rating_item, addDate, BTC, USD, EURO, sold, qLeft, shipFrom, shipTo, href, image, image_vendor) -#called by the crawler to get description links on a listing page -#@param: beautifulsoup object that is using the correct html page (listing page) -#return: list of description links from a listing page -def gofish_links_parser(soup): +# called by the crawler to get description links on a listing page +# @param: beautifulsoup object that is using the correct html page (listing page) +# return: list of description links from a listing page +def gofish_links_parser(soup): # Returning all links that should be visited by the Crawler href = [] - listing = soup.findAll('div', {"class": "card mt-1"}) + listing = soup.findAll('div', {"id": "itembox"}) + + # for a in listing: + # bae = a.find('a', {"class": "text-info"}, href=True) + # link = bae['href'] + # href.append(link) for a in listing: - bae = a.find('a', href=True)#card-title rounded text-truncate - link = bae['href'] + bae = a.findAll('a', href=True) + + # Adding the url to the list of urls + link = bae[0].get('href') href.append(link) return href \ No newline at end of file From 48b3fee56a60632a1790851d03a067294d5e3827 Mon Sep 17 00:00:00 2001 From: Joshua Date: Mon, 27 Nov 2023 18:19:49 -0800 Subject: [PATCH 2/5] finished crawler for GoFish --- MarketPlaces/GoFish/crawler_selenium.py | 43 +++++++++---------- MarketPlaces/GoFish/parser.py | 5 ++- MarketPlaces/Initialization/marketsList.txt | 2 +- MarketPlaces/Initialization/markets_mining.py | 3 ++ 4 files changed, 28 insertions(+), 25 deletions(-) diff --git a/MarketPlaces/GoFish/crawler_selenium.py b/MarketPlaces/GoFish/crawler_selenium.py index dbdbd50..42f7bfa 100644 --- a/MarketPlaces/GoFish/crawler_selenium.py +++ b/MarketPlaces/GoFish/crawler_selenium.py @@ -41,7 +41,7 @@ def startCrawling(): print(driver.current_url, e) closeDriver(driver) - new_parse(mktName, baseURL, True) + # new_parse(mktName, baseURL, True) # Returns the name of the website @@ -121,21 +121,20 @@ def login(driver): input("Press ENTER when CAPTCHA is complete and login page has loaded\n") # entering username and password into input boxes - usernameBox = driver.find_element(by=By.XPATH, value='//input[@name="username"]') + usernameBox = driver.find_element(by=By.XPATH, value='//*[@id="username"]') # Username here - usernameBox.send_keys('aliciamykeys') - passwordBox = driver.find_element(by=By.XPATH, value='//input[@name="password"]') + usernameBox.send_keys('itsmedio') + passwordBox = driver.find_element(by=By.XPATH, value='//*[@id="password"]') # Password here - passwordBox.send_keys('aliciawherearemykey$') - # session time - session_select = Select(driver.find_element(by=By.XPATH, value='/html/body/main/div/div/div/div/div/form/div[4]/div/div[2]/select')) - session_select.select_by_visible_text('Session 60min') + passwordBox.send_keys('DementedBed123-') + # submit + submit = driver.find_element(by=By.XPATH, value='/html/body/div[1]/div[2]/form/div[7]/input') + submit.click() - input("Press ENTER when CAPTCHA is completed and you exit the newsletter\n") # wait for listing page show up (This Xpath may need to change based on different seed url) WebDriverWait(driver, 100).until(EC.visibility_of_element_located( - (By.XPATH, '//*[@id="submit"]'))) + (By.XPATH, '/html/body/div/div[3]/div[2]/div[1]'))) def savePage(driver, page, url): @@ -176,17 +175,17 @@ def getInterestedLinks(): links = [] # Hosting and Security - links.append('http://gofishbybookb4a2kvviuygmwjqfxx7nqsovweogs2cxvqvexhe7edyd.onion/?c=84') + # links.append('http://gofishbybookb4a2kvviuygmwjqfxx7nqsovweogs2cxvqvexhe7edyd.onion/?c=84') # Exploits and Kits links.append('http://gofishbybookb4a2kvviuygmwjqfxx7nqsovweogs2cxvqvexhe7edyd.onion/?c=107') # Botnets and Malware - links.append('http://gofishbybookb4a2kvviuygmwjqfxx7nqsovweogs2cxvqvexhe7edyd.onion/?c=97') + # links.append('http://gofishbybookb4a2kvviuygmwjqfxx7nqsovweogs2cxvqvexhe7edyd.onion/?c=97') # Other Software - links.append('http://gofishbybookb4a2kvviuygmwjqfxx7nqsovweogs2cxvqvexhe7edyd.onion/?c=108') + # links.append('http://gofishbybookb4a2kvviuygmwjqfxx7nqsovweogs2cxvqvexhe7edyd.onion/?c=108') # Hacking Guide - links.append('http://gofishbybookb4a2kvviuygmwjqfxx7nqsovweogs2cxvqvexhe7edyd.onion/?c=129') + # links.append('http://gofishbybookb4a2kvviuygmwjqfxx7nqsovweogs2cxvqvexhe7edyd.onion/?c=129') # Fraud (mostly carding) - links.append('http://gofishbybookb4a2kvviuygmwjqfxx7nqsovweogs2cxvqvexhe7edyd.onion/?c=128') + # links.append('http://gofishbybookb4a2kvviuygmwjqfxx7nqsovweogs2cxvqvexhe7edyd.onion/?c=128') return links @@ -212,7 +211,6 @@ def crawlForum(driver): driver.refresh() html = driver.page_source savePage(driver, html, link) - list = productPages(html) for item in list: @@ -225,16 +223,17 @@ def crawlForum(driver): driver.back() # comment out - break + # break # comment out - if count == 1: - break + # if count == 1: + # break try: - link = driver.find_element(by=By.XPATH, value='//a[contains(text(), "Next")]').get_attribute('href') + link = driver.find_element(by=By.XPATH, value='/html/body/div/div[3]/div[2]/div[2]/nav/ul/li[3]/a').get_attribute('href') if link == "": raise NoSuchElementException + link = urlparse.urljoin(baseURL, str(link)) count += 1 except NoSuchElementException: @@ -249,14 +248,14 @@ def crawlForum(driver): # Returns 'True' if the link is Topic link, may need to change for every website def isDescriptionLink(url): - if 'item' in url: + if 'a=' in url: return True return False # Returns True if the link is a listingPage link, may need to change for every website def isListingLink(url): - if 'category=' in url: + if 'c=' in url: return True return False diff --git a/MarketPlaces/GoFish/parser.py b/MarketPlaces/GoFish/parser.py index 61cbd48..de10035 100644 --- a/MarketPlaces/GoFish/parser.py +++ b/MarketPlaces/GoFish/parser.py @@ -271,8 +271,9 @@ def gofish_links_parser(soup): # Returning all links that should be visited by the Crawler href = [] - listing = soup.findAll('div', {"id": "itembox"}) - + listing = soup.find('tbody', {'class': 'border border-2 align-middle'}) + listing = soup.findAll('tr') + listing = listing[1:] # for a in listing: # bae = a.find('a', {"class": "text-info"}, href=True) # link = bae['href'] diff --git a/MarketPlaces/Initialization/marketsList.txt b/MarketPlaces/Initialization/marketsList.txt index b656841..f81b343 100644 --- a/MarketPlaces/Initialization/marketsList.txt +++ b/MarketPlaces/Initialization/marketsList.txt @@ -1 +1 @@ -CypherMarketplace \ No newline at end of file +GoFish \ No newline at end of file diff --git a/MarketPlaces/Initialization/markets_mining.py b/MarketPlaces/Initialization/markets_mining.py index e5fe69a..5f14289 100644 --- a/MarketPlaces/Initialization/markets_mining.py +++ b/MarketPlaces/Initialization/markets_mining.py @@ -15,6 +15,7 @@ from MarketPlaces.ViceCity.crawler_selenium import crawler as crawlerViceCity from MarketPlaces.CypherMarketplace.crawler_selenium import crawler as crawlerCypher from MarketPlaces.PabloEscobarMarket.crawler_selenium import crawler as crawlerPabloEscobar from MarketPlaces.Ares.crawler_selenium import crawler as crawlerAres +from MarketPlaces.GoFish.crawler_selenium import crawler as crawlerGoFish import configparser import os @@ -110,5 +111,7 @@ if __name__ == '__main__': crawlerPabloEscobar() elif mkt == "Ares": crawlerAres() + elif mkt == "GoFish": + crawlerGoFish() print("\nScraping process completed!") From 77565cfa2293f65a2ca49dc8681054e71f683aed Mon Sep 17 00:00:00 2001 From: Joshua Date: Tue, 5 Dec 2023 13:21:22 -0800 Subject: [PATCH 3/5] gofish parser done. now testing --- MarketPlaces/GoFish/crawler_selenium.py | 22 +-- MarketPlaces/GoFish/parser.py | 126 +++++++++--------- MarketPlaces/Initialization/prepare_parser.py | 5 + 3 files changed, 82 insertions(+), 71 deletions(-) diff --git a/MarketPlaces/GoFish/crawler_selenium.py b/MarketPlaces/GoFish/crawler_selenium.py index 42f7bfa..7ce4cee 100644 --- a/MarketPlaces/GoFish/crawler_selenium.py +++ b/MarketPlaces/GoFish/crawler_selenium.py @@ -31,17 +31,17 @@ baseURL = 'http://gofishbybookb4a2kvviuygmwjqfxx7nqsovweogs2cxvqvexhe7edyd.onion def startCrawling(): mktName = getMKTName() - driver = getAccess() - - if driver != 'down': - try: - login(driver) - crawlForum(driver) - except Exception as e: - print(driver.current_url, e) - closeDriver(driver) - - # new_parse(mktName, baseURL, True) + # driver = getAccess() + # + # if driver != 'down': + # try: + # login(driver) + # crawlForum(driver) + # except Exception as e: + # print(driver.current_url, e) + # closeDriver(driver) + + new_parse(mktName, baseURL, True) # Returns the name of the website diff --git a/MarketPlaces/GoFish/parser.py b/MarketPlaces/GoFish/parser.py index de10035..b1e5c0e 100644 --- a/MarketPlaces/GoFish/parser.py +++ b/MarketPlaces/GoFish/parser.py @@ -2,7 +2,6 @@ __author__ = 'DarkWeb' # Here, we are importing the auxiliary functions to clean or convert data from MarketPlaces.Utilities.utilities import * - # Here, we are importing BeautifulSoup to search through the HTML tree from bs4 import BeautifulSoup @@ -37,67 +36,75 @@ def gofish_description_parser(soup): vendor_image = "-1" # 20 Vendor_Image # Finding Product Name - divmb = soup.findAll('div', {'class': "mb-1"}) + divmb = soup.find('div', {'class': "p-3 mb-1 fs-3 fw-bold border border-2 bg-white rounded"}) - name = divmb[0].text + name = divmb.text name = name.replace('\n', ' ') name = name.replace(",", "") name = name.strip() # Finding Vendor - vendor = divmb[1].find('a').text.strip() + vendor = soup.find('div', {'class': 'my-1'}).text.strip() # Finding Vendor Rating - temp = soup.find('div', {'class': ""}).text - temp = temp.split('(') - rating = temp[0].replace("Vendor's Review : ", "") - rating = rating.replace("%", "") - rating_vendor = rating.strip() + # temp = soup.find('div', {'class': ""}).text + # temp = temp.split('(') + # rating = temp[0].replace("Vendor's Review : ", "") + # rating = rating.replace("%", "") + # rating_vendor = rating.strip() # Finding the Product Rating and Number of Product Reviews - reviews = temp[2].replace(" review)", "") - reviews = reviews.strip() + # reviews = temp[2].replace(" review)", "") + # reviews = reviews.strip() - temp = temp[1].split(")") - rating = temp[1].replace("Product Review : ", "") - rating = rating.replace("%", "") - rating_item = rating.strip() + # temp = temp[1].split(")") + # rating = temp[1].replace("Product Review : ", "") + # rating = rating.replace("%", "") + # rating_item = rating.strip() # Finding Prices - USD = soup.find('div', {'class': "h3 text-primary"}).text.strip() + precios = soup.findall('td', {'class': "text-end text-nowrap"}) + USD = precios[0].text.strip().replace('$', '') # Finding the Product Category - pmb = soup.findAll('p', {'class': "mb-1"}) + # pmb = soup.findAll('p', {'class': "mb-1"}) - category = pmb[-1].text - category = category.replace("Category: ", "").strip() + # category = pmb[-1].text + # category = category.replace("Category: ", "").strip() # Finding the Product Quantity Available - left = divmb[-1].text - left = left.split(",", 1)[1] - left = left.replace("in stock", "") - left = left.strip() + # left = divmb[-1].text + # left = left.split(",", 1)[1] + # left = left.replace("in stock", "") + # left = left.strip() # Finding Number Sold - sold = divmb[-1].text - sold = sold.split(",", 1)[0] - sold = sold.replace("sold", "") - sold = sold.strip() + # sold = divmb[-1].text + # sold = sold.split(",", 1)[0] + # sold = sold.replace("sold", "") + # sold = sold.strip() # Finding Shipment Information (Origin) - pmb[0].text - shipFrom = shipFrom.replace("Ships from: ", "").strip() + origin = soup.findall('div', {'class': "p-3 mt-2 mb-3 border border-2 bg-white rounded"}) + origin = origin[0].text.strip() + + shipFrom = origin[1:].strip() # Finding Shipment Information (Destination) - pmb[1].text - shipTo = shipTo.replace("Ships to: ", "").strip() + dest = soup.findall('div', {'class': 'p-3 mb-3 overflow-auto border border-2 bg-white rounded'}) + dest = dest[-1].text.strip() + + shipTo = dest[1:].strip() # Finding the Product description - cardbody = soup.findAll('div', {'class': "card-body"}) - describe = cardbody[1].text.strip() + cardbody = soup.findAll('div', {'class': "p-3 mb-3 overflow-auto border border-2 bg-white rounded"}) + describe = cardbody[0].text + describe = describe.replace('\n', ' ') + describe = describe.strip() # Finding Product Image - image = soup.find('div', {'class': 'product-primary'}).find('img') + img = soup.findall('figure', {'class': 'image-feature'})[0] + image = img.find('img', {'class': 'image-block rounded'}) image = image.get('src') image = image.split('base64,')[-1] @@ -159,21 +166,20 @@ def gofish_listing_parser(soup): image_vendor = [] # 21 Vendor_Image href = [] # 22 Product_Links - listing = soup.findAll('div', {"id": "itembox"}) + listing = soup.find('tbody', {"class": "border border-2 align-middle"}).findall('tr') # Populating the Number of Products nm = len(listing) for a in listing: bae = a.findAll('a', href=True) - lb = a.findAll('div', {"id": "littlebox"}) # Adding the url to the list of urls link = bae[0].get('href') href.append(link) # Finding the Product - product = lb[1].find('a').text + product = bae[1].text product = product.replace('\n', ' ') product = product.replace(",", "") product = product.replace("...", "") @@ -181,19 +187,19 @@ def gofish_listing_parser(soup): name.append(product) # Finding Product Image - product_image = a.find('img') + product_image = bae[0].find('img') product_image = product_image.get('src') product_image = product_image.split('base64,')[-1] image.append(product_image) # Finding Prices - price = lb[-1].find('div', {"class": "mb-1"}).text + price = a.find('span', {"class": "fw-bold text-nowrap"}).text price = price.replace("$","") price = price.strip() USD.append(price) # Finding the Vendor - vendor_name = lb[-1].find("a").text + vendor_name = bae[-1].text vendor_name = vendor_name.replace(",", "") vendor_name = vendor_name.strip() vendor.append(vendor_name) @@ -201,35 +207,35 @@ def gofish_listing_parser(soup): image_vendor.append("-1") # Finding the Category - cat = lb[-1].find("span").text - cat = cat.replace("class:", "") - cat = cat.strip() - category.append(cat) + # cat = lb[-1].find("span").text + # cat = cat.replace("class:", "") + # cat = cat.strip() + # category.append(cat) - span = lb[0].findAll("span") + # span = lb[0].findAll("span") # Finding Number of Views - num = span[0].text - num = num.replace("views:", "") - num = num.strip() - sold.append(num) + # num = span[0].text + # num = num.replace("views:", "") + # num = num.strip() + # sold.append(num) # Finding Number Sold - num = span[2].text - num = num.replace("Sold:", "") - num = num.strip() - sold.append(num) + # num = span[2].text + # num = num.replace("Sold:", "") + # num = num.strip() + # sold.append(num) # Finding Quantity Left - quant = span[1].text - quant = quant.replace("stock:", "") - quant = quant.strip() - qLeft.append(quant) + # quant = span[1].text + # quant = quant.replace("stock:", "") + # quant = quant.strip() + # qLeft.append(quant) # add shipping information - ship = lb[2].findAll('small')[1].findAll('span')[1].text.split("->") - shipFrom.append(ship[0].replace("Ship from ", "").strip()) - shipTo.append(ship[1].replace("to ", "").strip()) + # ship = lb[2].findAll('small')[1].findAll('span')[1].text.split("->") + # shipFrom.append(ship[0].replace("Ship from ", "").strip()) + # shipTo.append(ship[1].replace("to ", "").strip()) # Searching for CVE and MS categories diff --git a/MarketPlaces/Initialization/prepare_parser.py b/MarketPlaces/Initialization/prepare_parser.py index 79a2bdc..4e3ab4a 100644 --- a/MarketPlaces/Initialization/prepare_parser.py +++ b/MarketPlaces/Initialization/prepare_parser.py @@ -19,6 +19,7 @@ from MarketPlaces.DarkBazar.parser import * from MarketPlaces.Sonanza.parser import * from MarketPlaces.Kingdom.parser import * from MarketPlaces.Ares.parser import * +from MarketPlaces.GoFish.parser import * from MarketPlaces.Classifier.classify_product import predict @@ -142,6 +143,8 @@ def parse_listing(marketPlace, listingFile, soup, createLog, logFile): rw = sonanza_listing_parser(soup) elif marketPlace == "Kingdom": rw = kingdom_listing_parser(soup) + elif marketPlace == "GoFish": + rw = gofish_listing_parser(soup) else: print("MISSING CALL TO LISTING PARSER IN PREPARE_PARSER.PY!") raise Exception @@ -184,6 +187,8 @@ def parse_description(marketPlace, descriptionFile, soup, createLog, logFile): rmm = sonanza_description_parser(soup) elif marketPlace == "Kingdom": rmm = kingdom_description_parser(soup) + elif marketPlace == "GoFish": + rmm = gofish_description_parser(soup) else: print("MISSING CALL TO DESCRIPTION PARSER IN PREPARE_PARSER.PY!") raise Exception From ac1eb6da97f0e7ec40c9b1162d0c26fa4b9aee46 Mon Sep 17 00:00:00 2001 From: Joshua Date: Wed, 6 Dec 2023 16:51:49 -0800 Subject: [PATCH 4/5] fixed Gofish parser --- MarketPlaces/GoFish/parser.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/MarketPlaces/GoFish/parser.py b/MarketPlaces/GoFish/parser.py index b1e5c0e..6efbae3 100644 --- a/MarketPlaces/GoFish/parser.py +++ b/MarketPlaces/GoFish/parser.py @@ -36,15 +36,20 @@ def gofish_description_parser(soup): vendor_image = "-1" # 20 Vendor_Image # Finding Product Name + divmb = soup.find('div', {'class': "p-3 mb-1 fs-3 fw-bold border border-2 bg-white rounded"}) + if divmb is None: + divmb = soup.find('div', {'class': "p-3 mb-1 fs-4 fw-bold border border-2 bg-white rounded"}) name = divmb.text name = name.replace('\n', ' ') + name = name.replace('\r', ' ') + name = name.replace('\t', ' ') name = name.replace(",", "") name = name.strip() # Finding Vendor - vendor = soup.find('div', {'class': 'my-1'}).text.strip() + vendor = soup.find('div', {'class': 'my-1'}).find('a').text.strip() # Finding Vendor Rating # temp = soup.find('div', {'class': ""}).text @@ -63,7 +68,7 @@ def gofish_description_parser(soup): # rating_item = rating.strip() # Finding Prices - precios = soup.findall('td', {'class': "text-end text-nowrap"}) + precios = soup.findAll('td', {'class': "text-end text-nowrap"}) USD = precios[0].text.strip().replace('$', '') # Finding the Product Category @@ -85,14 +90,18 @@ def gofish_description_parser(soup): # sold = sold.strip() # Finding Shipment Information (Origin) - origin = soup.findall('div', {'class': "p-3 mt-2 mb-3 border border-2 bg-white rounded"}) + origin = soup.findAll('div', {'class': "p-3 mt-2 mb-3 border border-2 bg-white rounded"}) + remove = origin[0].find('span').text.strip() origin = origin[0].text.strip() + origin = origin.replace(remove, '') - shipFrom = origin[1:].strip() + shipFrom = origin.strip() # Finding Shipment Information (Destination) - dest = soup.findall('div', {'class': 'p-3 mb-3 overflow-auto border border-2 bg-white rounded'}) + dest = soup.findAll('div', {'class': 'p-3 mb-3 overflow-auto border border-2 bg-white rounded'}) dest = dest[-1].text.strip() + dest = dest.replace('[', '') + dest = dest.replace(']', '') shipTo = dest[1:].strip() @@ -103,7 +112,7 @@ def gofish_description_parser(soup): describe = describe.strip() # Finding Product Image - img = soup.findall('figure', {'class': 'image-feature'})[0] + img = soup.findAll('figure', {'class': 'image-feature'})[0] image = img.find('img', {'class': 'image-block rounded'}) image = image.get('src') image = image.split('base64,')[-1] @@ -166,7 +175,7 @@ def gofish_listing_parser(soup): image_vendor = [] # 21 Vendor_Image href = [] # 22 Product_Links - listing = soup.find('tbody', {"class": "border border-2 align-middle"}).findall('tr') + listing = soup.find('tbody', {"class": "border border-2 align-middle"}).findAll('tr') # Populating the Number of Products nm = len(listing) From e6e8c594b51adcbb0c67dde91004e37fa44a6853 Mon Sep 17 00:00:00 2001 From: Joshua Date: Wed, 6 Dec 2023 16:53:33 -0800 Subject: [PATCH 5/5] uncommented gofish crawler --- MarketPlaces/GoFish/crawler_selenium.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/MarketPlaces/GoFish/crawler_selenium.py b/MarketPlaces/GoFish/crawler_selenium.py index 7ce4cee..19612cb 100644 --- a/MarketPlaces/GoFish/crawler_selenium.py +++ b/MarketPlaces/GoFish/crawler_selenium.py @@ -31,15 +31,15 @@ baseURL = 'http://gofishbybookb4a2kvviuygmwjqfxx7nqsovweogs2cxvqvexhe7edyd.onion def startCrawling(): mktName = getMKTName() - # driver = getAccess() - # - # if driver != 'down': - # try: - # login(driver) - # crawlForum(driver) - # except Exception as e: - # print(driver.current_url, e) - # closeDriver(driver) + driver = getAccess() + + if driver != 'down': + try: + login(driver) + crawlForum(driver) + except Exception as e: + print(driver.current_url, e) + closeDriver(driver) new_parse(mktName, baseURL, True)