__author__ = 'DarkWeb'
# Here, we are importing the auxiliary functions to clean or convert data
from MarketPlaces.Utilities.utilities import *
# Here, we are importing BeautifulSoup to search through the HTML tree
from bs4 import BeautifulSoup
# This is the method to parse the Description Pages (one page to each Product in the Listing Pages)
def kerberos_description_parser(soup):
# Fields to be parsed
name = "-1" # 0 Product_Name y
describe = "-1" # 1 Product_Description y
lastSeen = "-1" # 2 Product_LastViewDate
rules = "-1" # 3 NOT USED ...
CVE = "-1" # 4 Product_CVE_Classification (Common Vulnerabilities and Exposures)
MS = "-1" # 5 Product_MS_Classification (Microsoft Security)
review = "-1" # 6 Product_Number_Of_Reviews
category = "-1" # 7 Product_Category
shipFrom = "-1" # 8 Product_ShippedFrom
shipTo = "-1" # 9 Product_ShippedTo
left = "-1" # 10 Product_QuantityLeft y
escrow = "-1" # 11 Vendor_Warranty y
terms = "-1" # 12 Vendor_TermsAndConditions
vendor = "-1" # 13 Vendor_Name y
sold = "-1" # 14 Product_QuantitySold y
addDate = "-1" # 15 Product_AddedDate
available = "-1" # 16 NOT USED ...
endDate = "-1" # 17 NOT USED ...
BTC = "-1" # 18 Product_BTC_SellingPrice y
USD = "-1" # 19 Product_USD_SellingPrice y
rating = "-1" # 20 Vendor_Rating
success = "-1" # 21 Vendor_Successful_Transactions
EURO = "-1" # 22 Product_EURO_SellingPrice
bae = soup.find('div', {'class': "col-9"})
# Finding Product Name
name = bae.find('h2').text
name = name.replace('\n', ' ')
name = name.replace(",", "")
name = name.strip()
mb = bae.findAll('div', {"class": "mb-1"})
# Finding Vendor
vendor = mb[0].text
vendor = vendor.replace(",", "")
vendor = vendor.replace("Sold by:", "")
vendor = vendor.strip()
# # Finding Vendor Rating
# full_stars = bae[2].find_all('i', {'class': "fas fa-star"})
# half_star = bae[2].find('i', {'class': "fas fa-star-half-alt"})
# rating = len(full_stars) + (0.5 if half_star is not None else 0)
# Finding Warranty
escrow = mb[2].text
escrow = escrow.replace("Payment:", "")
escrow = escrow.strip()
# Finding Quantity Sold and Left
temp = mb[4].text.split(',')
sold = temp[0].replace("sold", "")
sold = sold.strip()
left = temp[1].replace("in stock", "")
left = left.strip()
# Finding USD
USD = bae.find('div', {"class": "h3 text-secondary"}).text
USD = USD.replace("$", "")
USD = USD.strip()
# Finding BTC
temp = bae.find('div', {"class": "small"}).text.split("BTC")
BTC = temp[0].strip()
# shipping_info = bae[4].text
# if "Digital" not in shipping_info:
# shipping_info = shipping_info.split(" ")
#
# # Finding Shipment Information (Origin)
# shipFrom = shipping_info[0].strip()
#
# # Finding Shipment Information (Destination)
# shipTo = shipping_info[1].strip()
# Finding the Product description
describe = bae.find('div', {"class": "card border-top-0"}).text
describe = describe.replace("\n", " ")
describe = describe.replace("\r", " ")
describe = describe.strip()
# Searching for CVE and MS categories
cve = soup.findAll(text=re.compile('CVE-\d{4}-\d{4}'))
if cve:
CVE = " "
for idx in cve:
CVE += (idx)
CVE += " "
CVE = CVE.replace(',', ' ')
CVE = CVE.replace('\n', '')
ms = soup.findAll(text=re.compile('MS\d{2}-\d{3}'))
if ms:
MS = " "
for im in ms:
MS += (im)
MS += " "
MS = MS.replace(',', ' ')
MS = MS.replace('\n', '')
# Populating the final variable (this should be a list with all fields scraped)
row = (name, describe, lastSeen, rules, CVE, MS, review, category, shipFrom, shipTo, left, escrow, terms, vendor,
sold, addDate, available, endDate, BTC, USD, rating, success, EURO)
# Sending the results
return row
# This is the method to parse the Listing Pages
def kerberos_listing_parser(soup):
# Fields to be parsed
nm = 0 # Total_Products (Should be Integer)
mktName = "Kerberos" # 0 Marketplace_Name
name = [] # 1 Product_Name y
CVE = [] # 2 Product_CVE_Classification (Common Vulnerabilities and Exposures)
MS = [] # 3 Product_MS_Classification (Microsoft Security)
category = [] # 4 Product_Category y
describe = [] # 5 Product_Description
escrow = [] # 6 Vendor_Warranty
views = [] # 7 Product_Number_Of_Views
reviews = [] # 8 Product_Number_Of_Reviews y
addDate = [] # 9 Product_AddDate
lastSeen = [] # 10 Product_LastViewDate
BTC = [] # 11 Product_BTC_SellingPrice
USD = [] # 12 Product_USD_SellingPrice y
EURO = [] # 13 Product_EURO_SellingPrice
sold = [] # 14 Product_QuantitySold
qLeft =[] # 15 Product_QuantityLeft
shipFrom = [] # 16 Product_ShippedFrom
shipTo = [] # 17 Product_ShippedTo
vendor = [] # 18 Vendor y
rating = [] # 19 Vendor_Rating
success = [] # 20 Vendor_Successful_Transactions
href = [] # 24 Product_Links (Urls)
listing = soup.findAll('div', {"class": "card product-card mb-3"})
# Populating the Number of Products
nm = len(listing)
# Finding Category
cat = soup.find("div", {"class": "col-9"})
cat = cat.find("h2").text
cat = cat.replace("Category: ", "")
cat = cat.replace(",", "")
cat = cat.strip()
for card in listing:
category.append(cat)
bae = card.findAll('a')
# Adding the url to the list of urls
link = bae[0].get('href')
link = cleanLink(link)
href.append(link)
# Finding Product Name
product = bae[1].text
product = product.replace('\n', ' ')
product = product.replace(",", "")
product = product.strip()
name.append(product)
# Finding Vendor
vendor_name = bae[2].text
vendor_name = vendor_name.replace(",", "")
vendor_name = vendor_name.strip()
vendor.append(vendor_name)
# Finding USD
usd = card.find('div', {"class": "mb-1"}).text
usd = usd.replace("$", "")
usd = usd.strip()
USD.append(usd)
# Finding Reviews
num = card.find("span", {"class": "rate-count"}).text
num = num.replace("(", "")
num = num.replace("review)", "")
num = num.replace("reviews)", "")
num = num.strip()
reviews.append(num)
# Searching for CVE and MS categories
cve = card.findAll(text=re.compile('CVE-\d{4}-\d{4}'))
if not cve:
cveValue="-1"
else:
cee = " "
for idx in cve:
cee += (idx)
cee += " "
cee = cee.replace(',', ' ')
cee = cee.replace('\n', '')
cveValue=cee
CVE.append(cveValue)
ms = card.findAll(text=re.compile('MS\d{2}-\d{3}'))
if not ms:
MSValue="-1"
else:
me = " "
for im in ms:
me += (im)
me += " "
me = me.replace(',', ' ')
me = me.replace('\n', '')
MSValue=me
MS.append(MSValue)
# Populate the final variable (this should be a list with all fields scraped)
return organizeProducts(mktName, nm, name, CVE, MS, category, describe, escrow, views, reviews, addDate, lastSeen,
BTC, USD, EURO, qLeft, shipFrom, shipTo, vendor, rating, success, sold, href)
def kerberos_links_parser(soup):
# Returning all links that should be visited by the Crawler
href = []
content = soup.find('div', {"id": "content-pos"})
listing = content.findAll('div', {"class": "item-block"})
for div in listing:
ae = div.find('div', {"ae zx300"})
links = ae.findAll('a')
href.append(links[1]['href'])
return href