|
|
- __author__ = 'DarkWeb'
-
- # Here, we are importing the auxiliary functions to clean or convert data
- from MarketPlaces.Utilities.utilities import *
-
- # Here, we are importing BeautifulSoup to search through the HTML tree
- from bs4 import BeautifulSoup
-
-
- # parses description pages, so takes html pages of description pages using soup object, and parses it for info it needs
- # stores info it needs in different lists, these lists are returned after being organized
- # @param: soup object looking at html page of description page
- # return: 'row' that contains a variety of lists that each hold info on the description page
- def pabloescobarmarket_description_parser(soup):
- # Fields to be parsed
-
- vendor = "-1" # 0 *Vendor_Name
- success = "-1" # 1 Vendor_Successful_Transactions
- rating_vendor = "-1" # 2 Vendor_Rating
- name = "-1" # 3 *Product_Name
- describe = "-1" # 4 Product_Description
- CVE = "-1" # 5 Product_CVE_Classification (Common Vulnerabilities and Exposures)
- MS = "-1" # 6 Product_MS_Classification (Microsoft Security)
- category = "-1" # 7 Product_Category
- views = "-1" # 8 Product_Number_Of_Views
- reviews = "-1" # 9 Product_Number_Of_Reviews
- rating_item = "-1" # 10 Product_Rating
- addDate = "-1" # 11 Product_AddedDate
- BTC = "-1" # 12 Product_BTC_SellingPrice
- USD = "-1" # 13 Product_USD_SellingPrice
- EURO = "-1" # 14 Product_EURO_SellingPrice
- sold = "-1" # 15 Product_QuantitySold
- left = "-1" # 16 Product_QuantityLeft
- shipFrom = "-1" # 17 Product_ShippedFrom
- shipTo = "-1" # 18 Product_ShippedTo
- image = "-1" # 19 Product_Image
- vendor_image = "-1" # 20 Vendor_Image
-
- # Finding Product Name
- # NA
-
- divmd7 = soup.find('div', {'class': "col-md-7"})
- ptag = soup.findAll('p')
- # Finding Vendor
- vendor = divmd7.find('a').text.strip()
-
- # Finding Vendor Rating
- # NA
-
- # Finding Successful Transactions
- success = soup.find('span', {'class': "badge-primary"})
-
- # Finding Prices
- USD = soup.find('span', {'class': "total"}).text.strip()
-
- BTC = soup.find('div', {'class': "text-center"}).text.strip()
-
- # Finding Escrow
- escrow = ptag[-1].text.strip()
-
- # Finding the Product Category
- category = ptag[-2].text.strip()
-
- # Finding the Product Quantity Available
- # NA
-
- # Finding Number Sold
- # NA
-
- # Finding Shipment Information (Origin)
- # NA
-
- # Finding Shipment Information (Destination)
- # NA
-
- # Finding the Product description
- describe = soup.find('div', {'class': "text-white"}).text
- describe = describe.replace("\n", " ")
- describe = describe.strip()
-
- '''# Finding the Number of Product Reviews
- tag = soup.findAll(text=re.compile('Reviews'))
- for index in tag:
- reviews = index
- par = reviews.find('(')
- if par >=0:
- reviews = reviews.replace("Reviews (","")
- reviews = reviews.replace(")","")
- reviews = reviews.split(",")
- review = str(abs(int(reviews[0])) + abs(int(reviews[1])))
- else :
- review = "-1"'''
-
- # Searching for CVE and MS categories
- cve = soup.findAll(text=re.compile('CVE-\d{4}-\d{4}'))
- if cve:
- CVE = " "
- for idx in cve:
- CVE += (idx)
- CVE += " "
- CVE = CVE.replace(',', ' ')
- CVE = CVE.replace('\n', '')
- ms = soup.findAll(text=re.compile('MS\d{2}-\d{3}'))
- if ms:
- MS = " "
- for im in ms:
- MS += (im)
- MS += " "
- MS = MS.replace(',', ' ')
- MS = MS.replace('\n', '')
-
- # Populating the final variable (this should be a list with all fields scraped)
- row = (vendor, rating_vendor, success, name, describe, CVE, MS, category, views, reviews, rating_item, addDate,
- BTC, USD, EURO, sold, left, shipFrom, shipTo, image, vendor_image)
-
- # Sending the results
- return row
-
-
- # parses listing pages, so takes html pages of listing pages using soup object, and parses it for info it needs
- # stores info it needs in different lists, these lists are returned after being organized
- # @param: soup object looking at html page of listing page
- # return: 'row' that contains a variety of lists that each hold info on the listing page
- def pabloescobarmarket_listing_parser(soup):
- # Fields to be parsed
- nm = 0 # *Total_Products (Should be Integer)
- mktName = "PabloEscobarMarket" # 0 *Marketplace_Name
- vendor = [] # 1 *Vendor y
- rating_vendor = [] # 2 Vendor_Rating
- success = [] # 3 Vendor_Successful_Transactions
- name = [] # 4 *Product_Name y
- CVE = [] # 5 Product_CVE_Classification (Common Vulnerabilities and Exposures) dont worry about this
- MS = [] # 6 Product_MS_Classification (Microsoft Security) dont worry about this
- category = [] # 7 Product_Category y
- describe = [] # 8 Product_Description
- views = [] # 9 Product_Number_Of_Views
- reviews = [] # 10 Product_Number_Of_Reviews
- rating_item = [] # 11 Product_Rating
- addDate = [] # 12 Product_AddDate
- BTC = [] # 13 Product_BTC_SellingPrice
- USD = [] # 14 Product_USD_SellingPrice y
- EURO = [] # 15 Product_EURO_SellingPrice
- sold = [] # 16 Product_QuantitySold
- qLeft = [] # 17 Product_QuantityLeft
- shipFrom = [] # 18 Product_ShippedFrom
- shipTo = [] # 19 Product_ShippedTo
- image = [] # 20 Product_Image
- image_vendor = [] # 21 Vendor_Image
- href = [] # 22 Product_Links
-
- listing = soup.findAll('div', {"class": "p-4"})
-
- # Populating the Number of Products
- nm = len(listing)
-
- for a in listing:
- bae = a.findAll('a', href=True)
-
- # Adding the url to the list of urls
- link = bae[0].get('href')
- link = cleanLink(link)
- href.append(link)
-
- # Finding the Product
- product = a.find('h4').text
- product = product.replace('\n', ' ')
- product = product.replace(",", "")
- product = product.replace("...", "")
- product = product.strip()
- name.append(product)
-
-
- # Finding Prices
- price = a.find('div', {"class": "price"}).text
- tempUSD = price.split("~")[0]
- tempUSD = tempUSD.replace("$", "")
- tempUSD = tempUSD.strip()
- USD.append(tempUSD)
-
- tempBTC = price.split("~")[1]
- tempBTC = tempBTC.replace("BTC", "")
- tempBTC = tempBTC.strip()
- BTC.append(tempBTC)
-
- # Finding the Vendor
- #NA
-
- # Finding the Category
- # NA
-
- # Finding Number Sold and Quantity Left
- # NA
-
- # Finding Successful Transactions
- # NA
-
- # Searching for CVE and MS categories
- cve = a.findAll(text=re.compile('CVE-\d{4}-\d{4}'))
- if not cve:
- cveValue = "-1"
- else:
- cee = " "
- for idx in cve:
- cee += (idx)
- cee += " "
- cee = cee.replace(',', ' ')
- cee = cee.replace('\n', '')
- cveValue = cee
- CVE.append(cveValue)
-
- ms = a.findAll(text=re.compile('MS\d{2}-\d{3}'))
- if not ms:
- MSValue = "-1"
- else:
- me = " "
- for im in ms:
- me += (im)
- me += " "
- me = me.replace(',', ' ')
- me = me.replace('\n', '')
- MSValue = me
- MS.append(MSValue)
-
- # Populate the final variable (this should be a list with all fields scraped)
- return organizeProducts(mktName, nm, vendor, rating_vendor, success, name, CVE, MS, category, describe, views,
- reviews, rating_item, addDate, BTC, USD, EURO, sold, qLeft, shipFrom, shipTo, href, image, image_vendor)
-
-
- # called by the crawler to get description links on a listing page
- # @param: beautifulsoup object that is using the correct html page (listing page)
- # return: list of description links from a listing page FIX
- def pabloescobarmarket_links_parser(soup):
- # Returning all links that should be visited by the Crawler
-
- href = []
- listing = soup.findAll('div', {"class": "p-4"})
-
- for a in listing:
- bae = a.find('a', href=True)
- link = bae['href']
- href.append(link)
-
- return href
|
