diff --git a/MarketPlaces/GoFish/crawler_selenium.py b/MarketPlaces/GoFish/crawler_selenium.py index e5af35b..dbdbd50 100644 --- a/MarketPlaces/GoFish/crawler_selenium.py +++ b/MarketPlaces/GoFish/crawler_selenium.py @@ -1,8 +1,7 @@ __author__ = 'DarkWeb' ''' -Go Fish market Crawler (Selenium) -- this is a new marketplace and was up for only a few days, crawler has not been finished +GoFish Marketplace Crawler (Selenium) ''' from selenium import webdriver @@ -11,6 +10,7 @@ from selenium.webdriver.firefox.firefox_profile import FirefoxProfile from selenium.webdriver.firefox.firefox_binary import FirefoxBinary from selenium.webdriver.firefox.service import Service from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support.ui import Select from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.common.by import By @@ -28,8 +28,7 @@ from MarketPlaces.Utilities.utilities import cleanHTML counter = 1 baseURL = 'http://gofishbybookb4a2kvviuygmwjqfxx7nqsovweogs2cxvqvexhe7edyd.onion/' -# Opens Tor Browser, crawls the website, then parses, then closes tor -#acts like the main method for the crawler, another function at the end of this code calls this function later + def startCrawling(): mktName = getMKTName() driver = getAccess() @@ -46,21 +45,18 @@ def startCrawling(): # Returns the name of the website -#return: name of site in string type def getMKTName(): name = 'GoFish' return name # Return the base link of the website -#return: url of base site in string type def getFixedURL(): url = 'http://gofishbybookb4a2kvviuygmwjqfxx7nqsovweogs2cxvqvexhe7edyd.onion/' return url # Closes Tor Browser -#@param: current selenium driver def closeDriver(driver): # global pid # os.system("taskkill /pid " + str(pro.pid)) @@ -110,7 +106,6 @@ def createFFDriver(): #the driver 'gets' the url, attempting to get on the site, if it can't access return 'down' -#return: return the selenium driver or string 'down' def getAccess(): url = getFixedURL() driver = createFFDriver() @@ -122,31 +117,27 @@ def getAccess(): return 'down' -# Manual captcha solver, waits fora specific element so that the whole page loads, finds the input box, gets screenshot of captcha -# then allows for manual solving of captcha in the terminal -#@param: current selenium web driver def login(driver): - - # wait for page to show up (This Xpath may need to change based on different seed url) - WebDriverWait(driver, 100).until(EC.visibility_of_element_located( - (By.XPATH, '//*[@id="username"]'))) + input("Press ENTER when CAPTCHA is complete and login page has loaded\n") # entering username and password into input boxes - usernameBox = driver.find_element(by=By.XPATH, value='//*[@id="username"]') + usernameBox = driver.find_element(by=By.XPATH, value='//input[@name="username"]') # Username here - usernameBox.send_keys('itsmedio') - passwordBox = driver.find_element(by=By.XPATH, value='//*[@id="password"]') + usernameBox.send_keys('aliciamykeys') + passwordBox = driver.find_element(by=By.XPATH, value='//input[@name="password"]') # Password here - passwordBox.send_keys('DementedBed123-') + passwordBox.send_keys('aliciawherearemykey$') + # session time + session_select = Select(driver.find_element(by=By.XPATH, value='/html/body/main/div/div/div/div/div/form/div[4]/div/div[2]/select')) + session_select.select_by_visible_text('Session 60min') - input("Press ENTER when CAPTCHA is completed and logged in\n") + input("Press ENTER when CAPTCHA is completed and you exit the newsletter\n") # wait for listing page show up (This Xpath may need to change based on different seed url) WebDriverWait(driver, 100).until(EC.visibility_of_element_located( - (By.XPATH, "/html/body/div[1]/div[3]/div[1]/div[3]/ul/div"))) + (By.XPATH, '//*[@id="submit"]'))) -# Saves the crawled html page, makes the directory path for html pages if not made def savePage(driver, page, url): cleanPage = cleanHTML(driver, page) filePath = getFullPathName(url) @@ -155,8 +146,6 @@ def savePage(driver, page, url): return -# Gets the full path of the page to be saved along with its appropriate file name -#@param: raw url as crawler crawls through every site def getFullPathName(url): from MarketPlaces.Initialization.markets_mining import config, CURRENT_DATE @@ -169,38 +158,41 @@ def getFullPathName(url): return fullPath -# Creates the file name from passed URL, gives distinct name if can't be made unique after cleaned -#@param: raw url as crawler crawls through every site +def getMKTName() -> str: + name = 'GoFish' + return name + + def getNameFromURL(url): global counter name = ''.join(e for e in url if e.isalnum()) - if (name == ''): + if name == '': name = str(counter) counter = counter + 1 return name -# returns list of urls, here is where you can list the different urls of interest, the crawler runs through this list -#in this example, there are a couple of categories some threads fall under such as -# Guides and Tutorials, Digital Products, and Software and Malware -#as you can see they are categories of products def getInterestedLinks(): links = [] - # Hacking - links.append('http://gofishbybookb4a2kvviuygmwjqfxx7nqsovweogs2cxvqvexhe7edyd.onion/?c=129') - # Malware - links.append('http://gofishbybookb4a2kvviuygmwjqfxx7nqsovweogs2cxvqvexhe7edyd.onion/?c=97') - # Exploits + # Hosting and Security + links.append('http://gofishbybookb4a2kvviuygmwjqfxx7nqsovweogs2cxvqvexhe7edyd.onion/?c=84') + # Exploits and Kits links.append('http://gofishbybookb4a2kvviuygmwjqfxx7nqsovweogs2cxvqvexhe7edyd.onion/?c=107') + # Botnets and Malware + links.append('http://gofishbybookb4a2kvviuygmwjqfxx7nqsovweogs2cxvqvexhe7edyd.onion/?c=97') + # Other Software + links.append('http://gofishbybookb4a2kvviuygmwjqfxx7nqsovweogs2cxvqvexhe7edyd.onion/?c=108') + # Hacking Guide + links.append('http://gofishbybookb4a2kvviuygmwjqfxx7nqsovweogs2cxvqvexhe7edyd.onion/?c=129') + # Fraud (mostly carding) + links.append('http://gofishbybookb4a2kvviuygmwjqfxx7nqsovweogs2cxvqvexhe7edyd.onion/?c=128') return links -# gets links of interest to crawl through, iterates through list, where each link is clicked and crawled through -#topic and description pages are crawled through here, where both types of pages are saved -#@param: selenium driver def crawlForum(driver): + print("Crawling the GoFish market") linksToCrawl = getInterestedLinks() @@ -222,6 +214,7 @@ def crawlForum(driver): savePage(driver, html, link) list = productPages(html) + for item in list: itemURL = urlparse.urljoin(baseURL, str(item)) try: @@ -229,7 +222,6 @@ def crawlForum(driver): except: driver.refresh() savePage(driver, driver.page_source, item) - time.sleep(3) driver.back() # comment out @@ -240,7 +232,7 @@ def crawlForum(driver): break try: - link = driver.find_element(by=By.LINK_TEXT, value='Next ›').get_attribute('href') + link = driver.find_element(by=By.XPATH, value='//a[contains(text(), "Next")]').get_attribute('href') if link == "": raise NoSuchElementException count += 1 @@ -252,30 +244,23 @@ def crawlForum(driver): print(link, e) i += 1 - print("Crawling the GoFishMarket done.") + print("Crawling the GoFish market done.") -# Returns 'True' if the link is a description link -#@param: url of any url crawled -#return: true if is a description page, false if not +# Returns 'True' if the link is Topic link, may need to change for every website def isDescriptionLink(url): - if 'c' in url: + if 'item' in url: return True return False -# Returns True if the link is a listingPage link -#@param: url of any url crawled -#return: true if is a Listing page, false if not +# Returns True if the link is a listingPage link, may need to change for every website def isListingLink(url): - if 'a' in url: + if 'category=' in url: return True return False -# calling the parser to define the links, the html is the url of a link from the list of interested link list -#@param: link from interested link list ie. getInterestingLinks() -#return: list of description links that should be crawled through def productPages(html): soup = BeautifulSoup(html, "html.parser") return gofish_links_parser(soup) @@ -283,4 +268,3 @@ def productPages(html): def crawler(): startCrawling() - # print("Crawling and Parsing BestCardingWorld .... DONE!") diff --git a/MarketPlaces/GoFish/parser.py b/MarketPlaces/GoFish/parser.py index 28e1a6b..61cbd48 100644 --- a/MarketPlaces/GoFish/parser.py +++ b/MarketPlaces/GoFish/parser.py @@ -7,321 +7,282 @@ from MarketPlaces.Utilities.utilities import * from bs4 import BeautifulSoup -#parses description pages, so takes html pages of description pages using soup object, and parses it for info it needs -#stores info it needs in different lists, these lists are returned after being organized -#@param: soup object looking at html page of description page -#return: 'row' that contains a variety of lists that each hold info on the description page +# parses description pages, so takes html pages of description pages using soup object, and parses it for info it needs +# stores info it needs in different lists, these lists are returned after being organized +# @param: soup object looking at html page of description page +# return: 'row' that contains a variety of lists that each hold info on the description page def gofish_description_parser(soup): - # Fields to be parsed - vendor = "-1" # 0 *Vendor_Name - success = "-1" # 1 Vendor_Successful_Transactions - rating_vendor = "-1" # 2 Vendor_Rating - name = "-1" # 3 *Product_Name - describe = "-1" # 4 Product_Description - CVE = "-1" # 5 Product_CVE_Classification (Common Vulnerabilities and Exposures) dont worry about that much - MS = "-1" # 6 Product_MS_Classification (Microsoft Security) dont worry about that much - category = "-1" # 7 Product_Category - views = "-1" # 8 Product_Number_Of_Views - reviews = "-1" # 9 Product_Number_Of_Reviews - rating_item = "-1" # 10 Product_Rating - addDate = "-1" # 11 Product_AddedDate - BTC = "-1" # 12 Product_BTC_SellingPrice - USD = "-1" # 13 Product_USD_SellingPrice - EURO = "-1" # 14 Product_EURO_SellingPrice - sold = "-1" # 15 Product_QuantitySold - left = "-1" # 16 Product_QuantityLeft - shipFrom = "-1" # 17 Product_ShippedFrom - shipTo = "-1" # 18 Product_ShippedTo - - #vendor name - try: - temp = soup.find('div', {'class': 'box rounded mb-0'}).find('a').text - vendor = (cleanString(temp.strip())) - except: - vendor = "-1" - - #successful transaction - try: - temp = soup.findAll('div', {'class','text-center text-truncate column-flex ml-1 mr-1'}) #card sidebar-menu mb-4 card sidebar-menu mb-4 - temp2 = temp[1].findAll('span', {'class', 'float-right font-weight-bold'}) - temp = temp2[1].text - success = (temp.strip()) - except: - print("success") - - - #vendor rating 5 - try: - temp = soup.findAll('div', {'class', 'text-center text-truncate column-flex ml-1 mr-1'}) # card sidebar-menu mb-4 card sidebar-menu mb-4 - temp2 = temp[1].findAll('span', {'class', 'float-right font-weight-bold'}) - temp = temp2[5].text - rating_vendor = (cleanString(temp.strip())) - except: - print("product") - - # product name - try: - temp = soup.find('h3', {'class', 'h3 rounded card-title'}).find('span').text - name = (cleanString(temp.strip())) - except: - temp = soup.find('h3', {'class', 'h3 rounded card-title'}).find('span').find("div").text - name = (cleanString(temp.strip())) - - # product description - describe = soup.find('div', {'class': "box rounded flex-fill"}).find('pre').text - if "\n" in describe: - describe = describe.replace("\n", " ") - describe = describe.replace("\r", " ") - describe = cleanString(describe.strip()) - - CVE = "-1" # 5 Product_CVE_Classification (Common Vulnerabilities and Exposures) dont worry about that much - MS = "-1" # 6 Product_MS_Classification (Microsoft Security) dont worry about that much - - # product category - try: - temp = soup.findAll('table', {'class', 'table table-hover'}) - temp2 = temp[1].find('tr').findAll('td') - temp = temp2[1].text - category = cleanString(temp.strip()) - except: - temp = soup.find('table', {'class', 'table table-hover'}) - temp2 = temp.find('tbody').find('tr').findAll('td') - temp = temp2[1].text - category = cleanString(temp.strip()) - - # product number of view - try: - temp = soup.find('div', {'class', 'box rounded mb-0'}) - temp2 = temp.findAll('i') - temp = temp2[2].text - views = cleanString((temp.strip())) - except: - print('Product number of view') - - reviews = "-1" # 9 Product_Number_Of_Reviews - rating_item = "-1" # 10 Product_Rating - addDate = "-1" # 11 Product_AddedDate - - #BTC selling price box box-rounded mt-2 - try: - temp = soup.find('div', {'class', 'box box-rounded mt-2'}) - temp2 = temp.findAll('i', {'class', 'float-right color-prices'}) - temp = temp2[1].text - BTC = cleanString((temp.strip())) - except: - try: - temp = soup.find('div', {'class', 'box box-rounded'}) - temp2 = temp.findAll('span', {'class', 'float-right color-prices'}) - temp = temp2[1].text - BTC = cleanString((temp.strip())) - except: - print("BTC") - - - # USD selling price - try: - temp = soup.find('div', {'class', 'box box-rounded mt-2'}) - temp2 = temp.findAll('center') - temp = temp2[1].find('i').text - if "$" in temp: - temp = temp.replace("$", "") - USD = cleanString((temp.strip())) - except: - try: - temp = soup.find('div', {'class', 'box box-rounded'}) - temp2 = temp.findAll('center') - temp = temp2[1].find('span').text - if "$" in temp: - temp = temp.replace("$", "") - USD = cleanString((temp.strip())) - except: - print("USD") - - EURO = "-1" # 14 Product_EURO_SellingPrice - - - # product sold - try: - temp = soup.find('div', {'class', 'box rounded mb-0'}) # card sidebar-menu mb-4 card sidebar-menu mb-4 - temp2 = temp.find('i') - temp = temp2.text - sold = (cleanString(temp.strip())) - # sold = "-1" - except: - print("product sold") - - # product quantatiy left ###ERRROR - try: - temp = soup.findAll('table', {'class', 'table table-hover'}) - temp2 = temp[1].findAll('tr') - temp3 = temp2[1].findAll('td') - temp = temp3[1].text - left = cleanString(temp.strip()) - except: - temp = soup.find('table', {'class', 'table table-hover'}) - temp2 = temp.findAll('tr') - temp3 = temp2[1].findAll('td') - temp = temp3[1].text - left = cleanString(temp.strip()) - - - shipFrom = "-1" # 17 Product_ShippedFrom - shipTo = "-1" # 18 Product_ShippedTo + + vendor = "-1" # 0 *Vendor_Name + success = "-1" # 1 Vendor_Successful_Transactions + rating_vendor = "-1" # 2 Vendor_Rating + name = "-1" # 3 *Product_Name + describe = "-1" # 4 Product_Description + CVE = "-1" # 5 Product_CVE_Classification (Common Vulnerabilities and Exposures) + MS = "-1" # 6 Product_MS_Classification (Microsoft Security) + category = "-1" # 7 Product_Category + views = "-1" # 8 Product_Number_Of_Views + reviews = "-1" # 9 Product_Number_Of_Reviews + rating_item = "-1" # 10 Product_Rating + addDate = "-1" # 11 Product_AddedDate + BTC = "-1" # 12 Product_BTC_SellingPrice + USD = "-1" # 13 Product_USD_SellingPrice + EURO = "-1" # 14 Product_EURO_SellingPrice + sold = "-1" # 15 Product_QuantitySold + left = "-1" # 16 Product_QuantityLeft + shipFrom = "-1" # 17 Product_ShippedFrom + shipTo = "-1" # 18 Product_ShippedTo + image = "-1" # 19 Product_Image + vendor_image = "-1" # 20 Vendor_Image + + # Finding Product Name + divmb = soup.findAll('div', {'class': "mb-1"}) + + name = divmb[0].text + name = name.replace('\n', ' ') + name = name.replace(",", "") + name = name.strip() + + # Finding Vendor + vendor = divmb[1].find('a').text.strip() + + # Finding Vendor Rating + temp = soup.find('div', {'class': ""}).text + temp = temp.split('(') + rating = temp[0].replace("Vendor's Review : ", "") + rating = rating.replace("%", "") + rating_vendor = rating.strip() + + # Finding the Product Rating and Number of Product Reviews + reviews = temp[2].replace(" review)", "") + reviews = reviews.strip() + + temp = temp[1].split(")") + rating = temp[1].replace("Product Review : ", "") + rating = rating.replace("%", "") + rating_item = rating.strip() + + # Finding Prices + USD = soup.find('div', {'class': "h3 text-primary"}).text.strip() + + # Finding the Product Category + pmb = soup.findAll('p', {'class': "mb-1"}) + + category = pmb[-1].text + category = category.replace("Category: ", "").strip() + + # Finding the Product Quantity Available + left = divmb[-1].text + left = left.split(",", 1)[1] + left = left.replace("in stock", "") + left = left.strip() + + # Finding Number Sold + sold = divmb[-1].text + sold = sold.split(",", 1)[0] + sold = sold.replace("sold", "") + sold = sold.strip() + + # Finding Shipment Information (Origin) + pmb[0].text + shipFrom = shipFrom.replace("Ships from: ", "").strip() + + # Finding Shipment Information (Destination) + pmb[1].text + shipTo = shipTo.replace("Ships to: ", "").strip() + + # Finding the Product description + cardbody = soup.findAll('div', {'class': "card-body"}) + describe = cardbody[1].text.strip() + + # Finding Product Image + image = soup.find('div', {'class': 'product-primary'}).find('img') + image = image.get('src') + image = image.split('base64,')[-1] + + # Searching for CVE and MS categories + cve = soup.findAll(text=re.compile('CVE-\d{4}-\d{4}')) + if cve: + CVE = " " + for idx in cve: + CVE += (idx) + CVE += " " + CVE = CVE.replace(',', ' ') + CVE = CVE.replace('\n', '') + ms = soup.findAll(text=re.compile('MS\d{2}-\d{3}')) + if ms: + MS = " " + for im in ms: + MS += (im) + MS += " " + MS = MS.replace(',', ' ') + MS = MS.replace('\n', '') # Populating the final variable (this should be a list with all fields scraped) row = (vendor, rating_vendor, success, name, describe, CVE, MS, category, views, reviews, rating_item, addDate, - BTC, USD, EURO, sold, left, shipFrom, shipTo) + BTC, USD, EURO, sold, left, shipFrom, shipTo, image, vendor_image) # Sending the results return row -#parses listing pages, so takes html pages of listing pages using soup object, and parses it for info it needs -#stores info it needs in different lists, these lists are returned after being organized -#@param: soup object looking at html page of listing page -#return: 'row' that contains a variety of lists that each hold info on the listing page +# parses listing pages, so takes html pages of listing pages using soup object, and parses it for info it needs +# stores info it needs in different lists, these lists are returned after being organized +# @param: soup object looking at html page of listing page +# return: 'row' that contains a variety of lists that each hold info on the listing page def gofish_listing_parser(soup): + # Fields to be parsed nm = 0 # *Total_Products (Should be Integer) - mktName = "GoFish" # 0 *Marketplace_Name - vendor = [] # 1 *Vendor y - rating_vendor = [] # 2 Vendor_Rating - success = [] # 3 Vendor_Successful_Transactions - name = [] # 4 *Product_Name y - CVE = [] # 5 Product_CVE_Classification (Common Vulnerabilities and Exposures) dont worry about this - MS = [] # 6 Product_MS_Classification (Microsoft Security) dont worry about this - category = [] # 7 Product_Category y - describe = [] # 8 Product_Description - views = [] # 9 Product_Number_Of_Views - reviews = [] # 10 Product_Number_Of_Reviews - rating_item = [] # 11 Product_Rating - addDate = [] # 12 Product_AddDate - BTC = [] # 13 Product_BTC_SellingPrice - USD = [] # 14 Product_USD_SellingPrice y - EURO = [] # 15 Product_EURO_SellingPrice - sold = [] # 16 Product_QuantitySold - qLeft = [] # 17 Product_QuantityLeft - shipFrom = [] # 18 Product_ShippedFrom - shipTo = [] # 19 Product_ShippedTo - href = [] # 20 Product_Links - - listing = soup.findAll('div', {"class": "card mt-1"}) + mktName = "GoFish" # 0 *Marketplace_Name + vendor = [] # 1 *Vendor y + rating_vendor = [] # 2 Vendor_Rating + success = [] # 3 Vendor_Successful_Transactions + name = [] # 4 *Product_Name y + CVE = [] # 5 Product_CVE_Classification (Common Vulnerabilities and Exposures) dont worry about this + MS = [] # 6 Product_MS_Classification (Microsoft Security) dont worry about this + category = [] # 7 Product_Category y + describe = [] # 8 Product_Description + views = [] # 9 Product_Number_Of_Views + reviews = [] # 10 Product_Number_Of_Reviews + rating_item = [] # 11 Product_Rating + addDate = [] # 12 Product_AddDate + BTC = [] # 13 Product_BTC_SellingPrice + USD = [] # 14 Product_USD_SellingPrice y + EURO = [] # 15 Product_EURO_SellingPrice + sold = [] # 16 Product_QuantitySold + qLeft = [] # 17 Product_QuantityLeft + shipFrom = [] # 18 Product_ShippedFrom + shipTo = [] # 19 Product_ShippedTo + image = [] # 20 Product_Image + image_vendor = [] # 21 Vendor_Image + href = [] # 22 Product_Links + + listing = soup.findAll('div', {"id": "itembox"}) # Populating the Number of Products nm = len(listing) for a in listing: + bae = a.findAll('a', href=True) + lb = a.findAll('div', {"id": "littlebox"}) - # vendor - try: - temp = a.find('div', {'class','col-5 justify-content-between mx-auto'}).find('a').text - vendor.append(cleanString(temp.strip())) - except: - print('vendor') - - #vendor rating - - - #successful transactions CHECK AGAIN HERE - try: - success.append("-1") - except: - print('successful transactions') - - # product name - try: - temp = a.find('h5', {'class','card-title rounded text-truncate'}).find('a').text - name.append(cleanString(temp.strip())) - except: - print('product name') - - - CVE.append('-1') - MS.append('-1') - rating_vendor.append('-1') - - # product category - try: - temp = soup.find('div', {'class', 'card-sidebar-menu box mb-2 flex-column'}).find('h3').find('span').text - if "Search Results for: " in temp: - temp = temp.replace("Search Results for: ", "") - category.append(cleanString(temp.strip())) - - except: - print("Error in product category") - - describe.append('-1') - - # product views - try: - temp = a.find('h6',{'class', 'card-subtitle mb-1 text-muted text-truncate'}) - temp2 = temp.find('i').text - views.append(cleanString(temp2.strip())) - except: - print("Error in views") - - reviews.append('-1') # 10 Product_Number_Of_Reviews - rating_item.append('-1') # 11 Product_Rating - addDate.append('-1') # 12 Product_AddDate - - # BTC - try: - temp = a.find('div', {'class', 'col-3 justify-content-between mx-auto'}) - temp2 = temp.findAll('p') - temp = temp2[1].text - BTC.append(cleanString(temp.strip())) - except: - print("BTC") - - #USD ERROR get rid of $ - try: - temp = a.find('div', {'class', 'col-12 justify-content-between mx-auto'}).find('i').text - if '$' in temp: - temp = temp.replace("$", "") - USD.append(cleanString(temp.strip())) # 14 Product_USD_SellingPrice - except: - print("USD") - - EURO.append("-1") # 15 Product_EURO_SellingPrice - - #product sold - try: - temp = a.find('div', {'class', 'col-12 mx-auto text-truncate text-center flex-fill'}).findAll('p', {'class', 'card-text mb-0'}) - temp2 = temp[1].find('i').text - sold.append(cleanString(temp2.strip())) - except: - print("product sold") - - qLeft.append('-1') # 17 Product_QuantityLeft - shipFrom.append('-1') # 18 Product_ShippedFrom - shipTo.append('-1') # 19 Product_ShippedTo - - #href - try: - temp = a.find('h5', {'class', 'card-title rounded text-truncate'}).find('a').get('href') - href.append(temp) # 20 Product_Links - except: - print("href") + # Adding the url to the list of urls + link = bae[0].get('href') + href.append(link) + # Finding the Product + product = lb[1].find('a').text + product = product.replace('\n', ' ') + product = product.replace(",", "") + product = product.replace("...", "") + product = product.strip() + name.append(product) + + # Finding Product Image + product_image = a.find('img') + product_image = product_image.get('src') + product_image = product_image.split('base64,')[-1] + image.append(product_image) + + # Finding Prices + price = lb[-1].find('div', {"class": "mb-1"}).text + price = price.replace("$","") + price = price.strip() + USD.append(price) + + # Finding the Vendor + vendor_name = lb[-1].find("a").text + vendor_name = vendor_name.replace(",", "") + vendor_name = vendor_name.strip() + vendor.append(vendor_name) + + image_vendor.append("-1") + + # Finding the Category + cat = lb[-1].find("span").text + cat = cat.replace("class:", "") + cat = cat.strip() + category.append(cat) + + span = lb[0].findAll("span") + + # Finding Number of Views + num = span[0].text + num = num.replace("views:", "") + num = num.strip() + sold.append(num) + + # Finding Number Sold + num = span[2].text + num = num.replace("Sold:", "") + num = num.strip() + sold.append(num) + + # Finding Quantity Left + quant = span[1].text + quant = quant.replace("stock:", "") + quant = quant.strip() + qLeft.append(quant) + + # add shipping information + ship = lb[2].findAll('small')[1].findAll('span')[1].text.split("->") + shipFrom.append(ship[0].replace("Ship from ", "").strip()) + shipTo.append(ship[1].replace("to ", "").strip()) + + + # Searching for CVE and MS categories + cve = a.findAll(text=re.compile('CVE-\d{4}-\d{4}')) + if not cve: + cveValue = "-1" + else: + cee = " " + for idx in cve: + cee += (idx) + cee += " " + cee = cee.replace(',', ' ') + cee = cee.replace('\n', '') + cveValue = cee + CVE.append(cveValue) + + ms = a.findAll(text=re.compile('MS\d{2}-\d{3}')) + if not ms: + MSValue = "-1" + else: + me = " " + for im in ms: + me += (im) + me += " " + me = me.replace(',', ' ') + me = me.replace('\n', '') + MSValue = me + MS.append(MSValue) # Populate the final variable (this should be a list with all fields scraped) return organizeProducts(mktName, nm, vendor, rating_vendor, success, name, CVE, MS, category, describe, views, - reviews, rating_item, addDate, BTC, USD, EURO, sold, qLeft, shipFrom, shipTo, href) + reviews, rating_item, addDate, BTC, USD, EURO, sold, qLeft, shipFrom, shipTo, href, image, image_vendor) -#called by the crawler to get description links on a listing page -#@param: beautifulsoup object that is using the correct html page (listing page) -#return: list of description links from a listing page -def gofish_links_parser(soup): +# called by the crawler to get description links on a listing page +# @param: beautifulsoup object that is using the correct html page (listing page) +# return: list of description links from a listing page +def gofish_links_parser(soup): # Returning all links that should be visited by the Crawler href = [] - listing = soup.findAll('div', {"class": "card mt-1"}) + listing = soup.findAll('div', {"id": "itembox"}) + + # for a in listing: + # bae = a.find('a', {"class": "text-info"}, href=True) + # link = bae['href'] + # href.append(link) for a in listing: - bae = a.find('a', href=True)#card-title rounded text-truncate - link = bae['href'] + bae = a.findAll('a', href=True) + + # Adding the url to the list of urls + link = bae[0].get('href') href.append(link) return href \ No newline at end of file