From 40e9bc10cf8caf3abacb20da41ad2215919de735 Mon Sep 17 00:00:00 2001 From: chris Date: Sun, 26 Nov 2023 10:23:29 -0800 Subject: [PATCH] Finished Crawler and Parser for MikesGrandStore --- .../MikesGrandStore/crawler_selenium.py | 118 +++--- MarketPlaces/MikesGrandStore/parser.py | 393 +++++++++--------- 2 files changed, 247 insertions(+), 264 deletions(-) diff --git a/MarketPlaces/MikesGrandStore/crawler_selenium.py b/MarketPlaces/MikesGrandStore/crawler_selenium.py index 2bb9e1d..492b306 100644 --- a/MarketPlaces/MikesGrandStore/crawler_selenium.py +++ b/MarketPlaces/MikesGrandStore/crawler_selenium.py @@ -1,7 +1,7 @@ -__author__ = 'Helium' +__author__ = 'cern' ''' -Mikes Grand Store Crawler (Selenium) +MikesGrandStore Marketplace Crawler (Selenium) ''' from selenium import webdriver @@ -10,6 +10,7 @@ from selenium.webdriver.firefox.firefox_profile import FirefoxProfile from selenium.webdriver.firefox.firefox_binary import FirefoxBinary from selenium.webdriver.firefox.service import Service from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support.ui import Select from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.common.by import By @@ -21,22 +22,21 @@ import subprocess import configparser from bs4 import BeautifulSoup from MarketPlaces.Initialization.prepare_parser import new_parse -from MarketPlaces.MikesGrandStore.parser import mikesgrandstore_links_parser +from MarketPlaces.MikesGrandStore.parser import MikesGrandStore_links_parser from MarketPlaces.Utilities.utilities import cleanHTML counter = 1 -baseURL = 'http://4yx2akutmkhwfgzlpdxiah7cknurw6vlddlq24fxa3r3ebophwgpvhyd.onion/' +baseURL = 'http://4yx2akutmkhwfgzlpdxiah7cknurw6vlddlq24fxa3r3ebophwgpvhyd.onion' -# Opens Tor Browser, crawls the website, then parses, then closes tor -#acts like the main method for the crawler, another function at the end of this code calls this function later def startCrawling(): mktName = getMKTName() driver = getAccess() if driver != 'down': try: - login(driver) + # Login not needed in MikesGrandStore + # login(driver) crawlForum(driver) except Exception as e: print(driver.current_url, e) @@ -46,21 +46,18 @@ def startCrawling(): # Returns the name of the website -#return: name of site in string type def getMKTName(): name = 'MikesGrandStore' return name # Return the base link of the website -#return: url of base site in string type def getFixedURL(): - url = 'http://4yx2akutmkhwfgzlpdxiah7cknurw6vlddlq24fxa3r3ebophwgpvhyd.onion/' + url = 'http://4yx2akutmkhwfgzlpdxiah7cknurw6vlddlq24fxa3r3ebophwgpvhyd.onion' return url # Closes Tor Browser -#@param: current selenium driver def closeDriver(driver): # global pid # os.system("taskkill /pid " + str(pro.pid)) @@ -86,8 +83,8 @@ def createFFDriver(): ff_prof.set_preference("privacy.sanitize.sanitizeOnShutdown", True) ff_prof.set_preference("signon.rememberSignons", False) ff_prof.set_preference("network.cookie.lifetimePolicy", 2) - ff_prof.set_preference("network.dns.disablePrefetch", True) - ff_prof.set_preference("network.http.sendRefererHeader", 0) + # ff_prof.set_preference("network.dns.disablePrefetch", True) + # ff_prof.set_preference("network.http.sendRefererHeader", 0) ff_prof.set_preference("permissions.default.image", 3) ff_prof.set_preference("browser.download.folderList", 2) ff_prof.set_preference("browser.download.manager.showWhenStarting", False) @@ -110,7 +107,6 @@ def createFFDriver(): #the driver 'gets' the url, attempting to get on the site, if it can't access return 'down' -#return: return the selenium driver or string 'down' def getAccess(): url = getFixedURL() driver = createFFDriver() @@ -122,16 +118,27 @@ def getAccess(): return 'down' -# Manual captcha solver, waits fora specific element so that the whole page loads, finds the input box, gets screenshot of captcha -# then allows for manual solving of captcha in the terminal -#@param: current selenium web driver def login(driver): - # wait for page to show up (This Xpath may need to change based on different seed url) + input("Press ENTER when CAPTCHA is complete and login page has loaded\n") + + # entering username and password into input boxes + usernameBox = driver.find_element(by=By.XPATH, value='//input[@name="username"]') + # Username here + usernameBox.send_keys('aliciamykeys') + passwordBox = driver.find_element(by=By.XPATH, value='//input[@name="password"]') + # Password here + passwordBox.send_keys('aliciawherearemykey$') + # session time + session_select = Select(driver.find_element(by=By.XPATH, value='/html/body/main/div/div/div/div/div/form/div[4]/div/div[2]/select')) + session_select.select_by_visible_text('Session 60min') + + input("Press ENTER when CAPTCHA is completed and you exit the newsletter\n") + + # wait for listing page show up (This Xpath may need to change based on different seed url) WebDriverWait(driver, 100).until(EC.visibility_of_element_located( - (By.XPATH, "/html/body/div[1]/header/div/div[3]/div/div/ul/li[6]/a"))) + (By.XPATH, '//*[@id="submit"]'))) -# Saves the crawled html page, makes the directory path for html pages if not made def savePage(driver, page, url): cleanPage = cleanHTML(driver, page) filePath = getFullPathName(url) @@ -140,50 +147,43 @@ def savePage(driver, page, url): return -# Gets the full path of the page to be saved along with its appropriate file name -#@param: raw url as crawler crawls through every site def getFullPathName(url): from MarketPlaces.Initialization.markets_mining import config, CURRENT_DATE mainDir = os.path.join(config.get('Project', 'shared_folder'), "MarketPlaces/" + getMKTName() + "/HTML_Pages") fileName = getNameFromURL(url) - if isDescriptionLink(url): - fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Description\\' + fileName + '.html') - else: + if isListingLink(url): fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Listing\\' + fileName + '.html') + else: + fullPath = os.path.join(mainDir, CURRENT_DATE + r'\\Description\\' + fileName + '.html') return fullPath -# Creates the file name from passed URL, gives distinct name if can't be made unique after cleaned -#@param: raw url as crawler crawls through every site +def getMKTName() -> str: + name = 'MikesGrandStore' + return name + + def getNameFromURL(url): global counter name = ''.join(e for e in url if e.isalnum()) - if (name == ''): + if name == '': name = str(counter) counter = counter + 1 return name -# returns list of urls, here is where you can list the different urls of interest, the crawler runs through this list -#in this example, there are a couple of categories some threads fall under such as -# Guides and Tutorials, Digital Products, and Software and Malware -#as you can see they are categories of products def getInterestedLinks(): links = [] - # Hacking and DDOS + # Hacking links.append('http://4yx2akutmkhwfgzlpdxiah7cknurw6vlddlq24fxa3r3ebophwgpvhyd.onion/product-category/hacking/') - # # databases - # links.append('http://4yx2akutmkhwfgzlpdxiah7cknurw6vlddlq24fxa3r3ebophwgpvhyd.onion/product-category/databases/') return links -# gets links of interest to crawl through, iterates through list, where each link is clicked and crawled through -#topic and description pages are crawled through here, where both types of pages are saved -#@param: selenium driver def crawlForum(driver): + print("Crawling the MikesGrandStore market") linksToCrawl = getInterestedLinks() @@ -205,6 +205,7 @@ def crawlForum(driver): savePage(driver, html, link) list = productPages(html) + for item in list: itemURL = urlparse.urljoin(baseURL, str(item)) try: @@ -215,15 +216,15 @@ def crawlForum(driver): driver.back() # comment out - break + #break # comment out - if count == 1: - break + #if count == 1: + # break + # go to next page try: - link = driver.find_element(by=By.XPATH, value= - '/html/body/div[1]/main/div/div[1]/div/div[3]/nav/ul/li[6]/a').get_attribute('href') + link = driver.find_element(by=By.XPATH, value="//a[@class='next page-number']").get_attribute('href') if link == "": raise NoSuchElementException count += 1 @@ -238,41 +239,28 @@ def crawlForum(driver): print("Crawling the MikesGrandStore market done.") -# Returns 'True' if the link is a description link -#@param: url of any url crawled -#return: true if is a description page, false if not +# Returns 'True' if the link is Topic link, may need to change for every website def isDescriptionLink(url): - if 'product/' in url: + if 'item' in url: return True return False -# Returns True if the link is a listingPage link -#@param: url of any url crawled -#return: true if is a Listing page, false if not +# Returns True if the link is a listingPage link, may need to change for every website def isListingLink(url): - if 'product-category' in url: + if 'category' in url: return True return False -# calling the parser to define the links, the html is the url of a link from the list of interested link list -#@param: link from interested link list ie. getInterestingLinks() -#return: list of description links that should be crawled through def productPages(html): soup = BeautifulSoup(html, "html.parser") - return mikesgrandstore_links_parser(soup) - - -# Drop links that "signout" -# def isSignOut(url): -# #absURL = urlparse.urljoin(url.base_url, url.url) -# if 'signout' in url.lower() or 'logout' in url.lower(): -# return True -# -# return False + return MikesGrandStore_links_parser(soup) def crawler(): startCrawling() - # print("Crawling and Parsing BestCardingWorld .... DONE!") + + +if __name__ == '__main__': + startCrawling() \ No newline at end of file diff --git a/MarketPlaces/MikesGrandStore/parser.py b/MarketPlaces/MikesGrandStore/parser.py index fe9bd61..1207eb2 100644 --- a/MarketPlaces/MikesGrandStore/parser.py +++ b/MarketPlaces/MikesGrandStore/parser.py @@ -1,223 +1,211 @@ __author__ = 'DarkWeb' # Here, we are importing the auxiliary functions to clean or convert data -from typing import List, Tuple from MarketPlaces.Utilities.utilities import * # Here, we are importing BeautifulSoup to search through the HTML tree from bs4 import BeautifulSoup -def mikesGrandStore_description_parser(soup: BeautifulSoup) -> Tuple: - - name = "-1" # 0 Product_Name - describe = "-1" # 1 Product_Description - lastSeen = "-1" # 2 Product_LastViewDate - rules = "-1" # 3 NOT USED ... - CVE = "-1" # 4 Product_CVE_Classification (Common Vulnerabilities and Exposures) - MS = "-1" # 5 Product_MS_Classification (Microsoft Security) - review = "-1" # 6 Product_Number_Of_Reviews +# parses description pages, so takes html pages of description pages using soup object, and parses it for info it needs +# stores info it needs in different lists, these lists are returned after being organized +# @param: soup object looking at html page of description page +# return: 'row' that contains a variety of lists that each hold info on the description page +def MikesGrandStore_description_parser(soup): + # Fields to be parsed + + vendor = "-1" # 0 *Vendor_Name + success = "-1" # 1 Vendor_Successful_Transactions + rating_vendor = "-1" # 2 Vendor_Rating + name = "-1" # 3 *Product_Name + describe = "-1" # 4 Product_Description + CVE = "-1" # 5 Product_CVE_Classification (Common Vulnerabilities and Exposures) + MS = "-1" # 6 Product_MS_Classification (Microsoft Security) category = "-1" # 7 Product_Category - shipFrom = "-1" # 8 Product_ShippedFrom - shipTo = "-1" # 9 Product_ShippedTo - left = "-1" # 10 Product_QuantityLeft - escrow = "-1" # 11 Vendor_Warranty - terms = "-1" # 12 Vendor_TermsAndConditions - vendor = "-1" # 13 Vendor_Name - sold = "-1" # 14 Product_QuantitySold - addDate = "-1" # 15 Product_AddedDate - available = "-1" # 16 NOT USED ... - endDate = "-1" # 17 NOT USED ... - BTC = "-1" # 18 Product_BTC_SellingPrice - USD = "-1" # 19 Product_USD_SellingPrice - rating = "-1" # 20 Vendor_Rating - success = "-1" # 21 Vendor_Successful_Transactions - EURO = "-1" # 22 Product_EURO_SellingPrice - - - name: str = soup.find("h1", {"class": "product-title product_title entry-title"}).text - - describe = soup.find("div", {"id": "tab-description"}).text - - commentsList: List[BeautifulSoup] = soup.find("ol", {"class": "commentlist"}).find_all("li") - - if len(commentsList) > 0: - lastReview: BeautifulSoup = commentsList[0] - lastSeen = lastReview.find("time").get("datetime").text - - reviewTab: str = soup.find('a', {'href': '#tab-reivews'}).text - review = reviewTab.split('(')[1].split(')')[0] - - navbarBreadcrumbs: List[BeautifulSoup] = soup.find('nav', {'class': 'woocommerce-breadcrumb breadcrumbs '}).find_all('a') - category = navbarBreadcrumbs[1].text - - USD = soup.find("div", {"class": "price-wrapper"}).text - - reviewStats: str = soup.find("div", {"class": "star-rating"}).text - rating = reviewStats.split(' ')[1] - - row = ( - name, - describe, - lastSeen, - rules, - CVE, - MS, - review, - category, - shipFrom, - shipTo, - left, - escrow, - terms, - vendor, - sold, - addDate, - available, - endDate, - BTC, - USD, - rating, - success, - EURO - ) - + views = "-1" # 8 Product_Number_Of_Views + reviews = "-1" # 9 Product_Number_Of_Reviews + rating_item = "-1" # 10 Product_Rating + addDate = "-1" # 11 Product_AddedDate + BTC = "-1" # 12 Product_BTC_SellingPrice + USD = "-1" # 13 Product_USD_SellingPrice + EURO = "-1" # 14 Product_EURO_SellingPrice + sold = "-1" # 15 Product_QuantitySold + left = "-1" # 16 Product_QuantityLeft + shipFrom = "-1" # 17 Product_ShippedFrom + shipTo = "-1" # 18 Product_ShippedTo + image = "-1" # 19 Product_Image + vendor_image = "-1" # 20 Vendor_Image + + # Finding Product Name + name = soup.find('h1', {'class': 'product-title product_title entry-title'}).text + name = name.replace('\n', ' ') + name = name.replace(",", "") + name = name.strip() + + divmb = soup.findAll('div', {'class': "mb-1"}) + + # Finding Vendor + # no vendor + vendor = "MikesGrandStore" + + # Finding the Product Rating + rating_item = soup.find('strong', {'class', 'rating'}).text + rating_item = rating_item.replace('\n', ' ') + rating_item = rating_item.replace(",", "") + rating_item = rating_item.strip() + + # Finding Number of Product Reviews + review_container = soup.find('li', {'id': 'tab-title-reviews'}) + reviews = review_container.find('a').text + reviews = reviews.replace('Reviews', '') + reviews = reviews.replace('(', '') + reviews = reviews.replace(')', '') + reviews = reviews.replace('\n', ' ') + reviews = reviews.replace(",", "") + reviews = reviews.strip() + + # Finding Prices + USD = soup.find('span', {'class': 'woocommerce-Price-currencySymbol'}).next_sibling + USD = USD.replace('\n', ' ') + USD = USD.replace(",", "") + USD = USD.strip() + + # Finding the Product Category + cat_container = soup.find('span', {'class': 'posted_in'}) + cat = cat_container.findAll('a') + category = "" + for name in cat: + category = category + " " + name.text + + # Finding the Product Quantity Available + stock = soup.find('p', {'class': 'stock in-stock'}) + if stock is not None: + left = stock.text + left = left.replace("in stock", "") + left = left.strip() + + # Finding the Product description + desc_cont = soup.find('div', {'class': 'product-short-description'}) + describe = desc_cont.find('p').text.strip() + + # Finding Product Image + image = soup.find('img', {'class': 'wp-post-image skip-lazy'}) + image = image.get('src') + image = image.split('base64,')[-1] + + # Searching for CVE and MS categories + cve = soup.findAll(text=re.compile('CVE-\d{4}-\d{4}')) + if cve: + CVE = " " + for idx in cve: + CVE += (idx) + CVE += " " + CVE = CVE.replace(',', ' ') + CVE = CVE.replace('\n', '') + ms = soup.findAll(text=re.compile('MS\d{2}-\d{3}')) + if ms: + MS = " " + for im in ms: + MS += (im) + MS += " " + MS = MS.replace(',', ' ') + MS = MS.replace('\n', '') + + # Populating the final variable (this should be a list with all fields scraped) + row = (vendor, rating_vendor, success, name, describe, CVE, MS, category, views, reviews, rating_item, addDate, + BTC, USD, EURO, sold, left, shipFrom, shipTo, image, vendor_image) + + # Sending the results return row -def mikesGrandStore_listing_parser(soup: BeautifulSoup) -> List: - - # Fields to be parsed - nm = 0 # Total_Products (Should be Integer) - mktName = "MikesGrandStore" # 0 Marketplace_Name - name = [] # 1 Product_Name - CVE = [] # 2 Product_CVE_Classification (Common Vulnerabilities and Exposures) - MS = [] # 3 Product_MS_Classification (Microsoft Security) - category = [] # 4 Product_Category - describe = [] # 5 Product_Description - escrow = [] # 6 Vendor_Warranty - views = [] # 7 Product_Number_Of_Views - reviews = [] # 8 Product_Number_Of_Reviews - addDate = [] # 9 Product_AddDate - lastSeen = [] # 10 Product_LastViewDate - BTC = [] # 11 Product_BTC_SellingPrice - USD = [] # 12 Product_USD_SellingPrice - EURO = [] # 13 Product_EURO_SellingPrice - sold = [] # 14 Product_QuantitySold - qLeft =[] # 15 Product_QuantityLeft - shipFrom = [] # 16 Product_ShippedFrom - shipTo = [] # 17 Product_ShippedTo - vendor = [] # 18 Vendor - rating = [] # 19 Vendor_Rating - success = [] # 20 Vendor_Successful_Transactions - href = [] # 23 Product_Links (Urls) - - - - pass - -#parses listing pages, so takes html pages of listing pages using soup object, and parses it for info it needs -#stores info it needs in different lists, these lists are returned after being organized -#@param: soup object looking at html page of listing page -#return: 'row' that contains a variety of lists that each hold info on the listing page -def darkfox_listing_parser(soup): +# parses listing pages, so takes html pages of listing pages using soup object, and parses it for info it needs +# stores info it needs in different lists, these lists are returned after being organized +# @param: soup object looking at html page of listing page +# return: 'row' that contains a variety of lists that each hold info on the listing page +def MikesGrandStore_listing_parser(soup): # Fields to be parsed - nm = 0 # Total_Products (Should be Integer) - mktName = "DarkFox" # 0 Marketplace_Name - name = [] # 1 Product_Name - CVE = [] # 2 Product_CVE_Classification (Common Vulnerabilities and Exposures) - MS = [] # 3 Product_MS_Classification (Microsoft Security) - category = [] # 4 Product_Category - describe = [] # 5 Product_Description - escrow = [] # 6 Vendor_Warranty - views = [] # 7 Product_Number_Of_Views - reviews = [] # 8 Product_Number_Of_Reviews - addDate = [] # 9 Product_AddDate - lastSeen = [] # 10 Product_LastViewDate - BTC = [] # 11 Product_BTC_SellingPrice - USD = [] # 12 Product_USD_SellingPrice - EURO = [] # 13 Product_EURO_SellingPrice - sold = [] # 14 Product_QuantitySold - qLeft =[] # 15 Product_QuantityLeft - shipFrom = [] # 16 Product_ShippedFrom - shipTo = [] # 17 Product_ShippedTo - vendor = [] # 18 Vendor - rating = [] # 19 Vendor_Rating - success = [] # 20 Vendor_Successful_Transactions - href = [] # 23 Product_Links (Urls) - - listing = soup.findAll('div', {"class": "card"}) + nm = 0 # *Total_Products (Should be Integer) + mktName = "MikesGrandStore" # 0 *Marketplace_Name + vendor = [] # 1 *Vendor y + rating_vendor = [] # 2 Vendor_Rating + success = [] # 3 Vendor_Successful_Transactions + name = [] # 4 *Product_Name y + CVE = [] # 5 Product_CVE_Classification (Common Vulnerabilities and Exposures) dont worry about this + MS = [] # 6 Product_MS_Classification (Microsoft Security) dont worry about this + category = [] # 7 Product_Category y + describe = [] # 8 Product_Description + views = [] # 9 Product_Number_Of_Views + reviews = [] # 10 Product_Number_Of_Reviews + rating_item = [] # 11 Product_Rating + addDate = [] # 12 Product_AddDate + BTC = [] # 13 Product_BTC_SellingPrice + USD = [] # 14 Product_USD_SellingPrice y + EURO = [] # 15 Product_EURO_SellingPrice + sold = [] # 16 Product_QuantitySold + qLeft = [] # 17 Product_QuantityLeft + shipFrom = [] # 18 Product_ShippedFrom + shipTo = [] # 19 Product_ShippedTo + image = [] # 20 Product_Image + image_vendor = [] # 21 Vendor_Image + href = [] # 22 Product_Links + + listing_container = soup.find('div', {'class': 'products row row-small large-columns-3 medium-columns-3 small-columns-2 equalize-box'}) + listing = listing_container.findAll('div', recursive=False) # Populating the Number of Products nm = len(listing) for a in listing: bae = a.findAll('a', href=True) + lb = a.findAll('div', {"id": "littlebox"}) # Adding the url to the list of urls - link = bae[0].get('href') - link = cleanLink(link) + link = a.find('a', {'class': 'woocommerce-LoopProduct-link woocommerce-loop-product__link'}).get('href') href.append(link) # Finding the Product - product = bae[1].find('p').text + product = a.find('a', {'class': 'woocommerce-LoopProduct-link woocommerce-loop-product__link'}).text product = product.replace('\n', ' ') product = product.replace(",", "") product = product.replace("...", "") product = product.strip() name.append(product) - bae = a.find('div', {'class': "media-content"}).find('div').find_all('div') - - if len(bae) >= 5: - # Finding Prices - price = bae[0].text - ud = price.replace(" USD", " ") - # u = ud.replace("$","") - u = ud.replace(",", "") - u = u.strip() - USD.append(u) - # bc = (prc[1]).strip(' BTC') - # BTC.append(bc) - - # Finding the Vendor - vendor_name = bae[1].find('a').text - vendor_name = vendor_name.replace(",", "") - vendor_name = vendor_name.strip() - vendor.append(vendor_name) - - # Finding the Category - cat = bae[2].find('small').text - cat = cat.replace("Category: ", "") - cat = cat.replace(",", "") - cat = cat.strip() - category.append(cat) - - # Finding Number Sold and Quantity Left - num = bae[3].text - num = num.replace("Sold: ", "") - num = num.strip() - sold.append(num) - - quant = bae[4].find('small').text - quant = quant.replace("In stock: ", "") - quant = quant.strip() - qLeft.append(quant) - - # Finding Successful Transactions - freq = bae[1].text - freq = freq.replace(vendor_name, "") - freq = re.sub(r'Vendor Level \d+', "", freq) - freq = freq.replace("(", "") - freq = freq.replace(")", "") - freq = freq.strip() - success.append(freq) + # Finding Product Image + product_image = a.find('img', {'class': 'attachment-woocommerce_thumbnail size-woocommerce_thumbnail'}) + product_image = product_image.get('src') + product_image = product_image.split('base64,')[-1] + image.append(product_image) + + # Finding Prices + price = a.find('span', {'class': 'woocommerce-Price-currencySymbol'}).next_sibling + price = price.strip() + USD.append(price) + + # Finding the Vendor + vendor_name = "MikesGrandStore" + vendor.append(vendor_name) + + image_vendor.append("-1") + + # Finding the Category + cat = a.find('p', {'class': 'category uppercase is-smaller no-text-overflow product-cat op-7'}).text + cat = cat.replace("class:", "") + cat = cat.strip() + category.append(cat) + + # Finding product rating + rating = a.find('strong', {'class': 'rating'}).text + rating = rating.strip() + rating_item.append(rating) + # Searching for CVE and MS categories cve = a.findAll(text=re.compile('CVE-\d{4}-\d{4}')) if not cve: - cveValue="-1" + cveValue = "-1" else: cee = " " for idx in cve: @@ -225,12 +213,12 @@ def darkfox_listing_parser(soup): cee += " " cee = cee.replace(',', ' ') cee = cee.replace('\n', '') - cveValue=cee + cveValue = cee CVE.append(cveValue) - + ms = a.findAll(text=re.compile('MS\d{2}-\d{3}')) if not ms: - MSValue="-1" + MSValue = "-1" else: me = " " for im in ms: @@ -238,27 +226,34 @@ def darkfox_listing_parser(soup): me += " " me = me.replace(',', ' ') me = me.replace('\n', '') - MSValue=me + MSValue = me MS.append(MSValue) # Populate the final variable (this should be a list with all fields scraped) - return organizeProducts(mktName, nm, name, CVE, MS, category, describe, escrow, views, reviews, addDate, lastSeen, - BTC, USD, EURO, qLeft, shipFrom, shipTo, vendor, rating, success, sold, href) - + return organizeProducts(mktName, nm, vendor, rating_vendor, success, name, CVE, MS, category, describe, views, + reviews, rating_item, addDate, BTC, USD, EURO, sold, qLeft, shipFrom, shipTo, href, image, image_vendor) -#called by the crawler to get description links on a listing page -#@param: beautifulsoup object that is using the correct html page (listing page) -#return: list of description links from a listing page -def mikesgrandstore_links_parser(soup): +# called by the crawler to get description links on a listing page +# @param: beautifulsoup object that is using the correct html page (listing page) +# return: list of description links from a listing page +def MikesGrandStore_links_parser(soup): # Returning all links that should be visited by the Crawler href = [] - listing = soup.findAll('div', {"class": "box-image"}) + container = soup.find('div', {"class": "products row row-small large-columns-3 medium-columns-3 small-columns-2 equalize-box"}) + listing = container.findAll('div', recursive=False) + + # for a in listing: + # bae = a.find('a', {"class": "text-info"}, href=True) + # link = bae['href'] + # href.append(link) for a in listing: - bae = a.find('div', {"class": "image-fade_in_back"}).find('a', href=True) - link = bae['href'] + bae = a.findAll('a', href=True) + + # Adding the url to the list of urls + link = bae[0].get('href') href.append(link) - return href \ No newline at end of file + return href