Browse Source

add parser

main
Nathan Pham 1 year ago
parent
commit
0de7cf5626
1 changed files with 98 additions and 88 deletions
  1. +98
    -88
      MarketPlaces/MetaVerseMarket/parser.py

+ 98
- 88
MarketPlaces/MetaVerseMarket/parser.py View File

@ -11,7 +11,7 @@ from bs4 import BeautifulSoup
# stores info it needs in different lists, these lists are returned after being organized # stores info it needs in different lists, these lists are returned after being organized
# @param: soup object looking at html page of description page # @param: soup object looking at html page of description page
# return: 'row' that contains a variety of lists that each hold info on the description page # return: 'row' that contains a variety of lists that each hold info on the description page
def darkfox_description_parser(soup): def metaversemarket_description_parser(soup):
# Fields to be parsed # Fields to be parsed
name = "-1" # 0 Product_Name name = "-1" # 0 Product_Name
@ -36,67 +36,58 @@ def darkfox_description_parser(soup):
EURO = "-1" # 22 Product_EURO_SellingPrice EURO = "-1" # 22 Product_EURO_SellingPrice
# Finding Product Name # Finding Product Name
name = soup.find('h1').text name = soup.find('div', {'class': "panel-heading"}).text.strip
name = name.replace('\n', ' ')
name = name.replace(",", "")
name = name.strip()
# Finding Vendor # Finding Vendor
vendor = soup.find('h3').find('a').text.strip() temp = soup.findAll('div', {'class': "col-xs-12 col-sm-6 mt-5"})
temp = temp[1].findAll('span')
temp = temp[1].find('b').text
name = temp.replace("@", "")
# Finding Vendor Rating # Finding Product Reviews
rating = soup.find('span', {'class': "tag is-dark"}).text.strip() review = soup.find('span', {'class': "badge bg-success fs-12px"}).text.strip()
# Finding Successful Transactions # Finding Successful Transactions
success = soup.find('h3').text # NA
success = success.replace("Vendor: ", "")
success = success.replace(vendor, "")
success = success.replace("(", "")
success = success.replace(")", "")
success = success.strip()
bae = soup.find('div', {'class': "box"}).find_all('ul')
# Finding Prices # Finding Prices
USD = bae[1].find('strong').text.strip() USD = soup.find('h3', {'class': "mb-2"}).text()
USD = USD.replace("Price: $", "").strip()
li = bae[2].find_all('li')
# Finding Escrow # Finding Escrow
escrow = li[0].find('span', {'class': "tag is-dark"}).text.strip() escrow = soup.find('div', {'class': "alert alert-info text-center fw-bold"}).text
escrow = escrow.replace('You are protected by ', "").strip()
# Finding the Product Category # Finding the Product Category
category = li[1].find('span', {'class': "tag is-dark"}).text.strip() temp = soup.select('div[class="mt-2"]')
temp = temp[0].findAll('span')
category = temp[1].text.strip()
# Finding the Product Quantity Available # Finding the Product Quantity Available
left = li[3].find('span', {'class': "tag is-dark"}).text.strip() # temp = soup.find('em', {'class': "icon ni ni-layers-fill"}).parent.parent.parent
# left = temp.text
# left = left.replace("Supply:", "")
# left = left.strip()
temp = soup.findAll('span', {'class': "badge bg-success"})
temp = temp[1].text.split("/")
left = temp[1].strip()
# Finding Number Sold # Finding Number Sold
sold = li[4].find('span', {'class': "tag is-dark"}).text.strip() sold = temp[0].strip()
li = bae[3].find_all('li')
# Finding Shipment Information (Origin) # Finding Shipment Information (Origin)
if "Ships from:" in li[-2].text: temp = soup.findAll('div', {'class': "alert alert-info"})
shipFrom = li[-2].text temp = temp[1].text.split("to")
shipFrom = shipFrom.replace("Ships from: ", "") shipFrom = temp[0].replace("Shipping from ", "").strip()
# shipFrom = shipFrom.replace(",", "")
shipFrom = shipFrom.strip()
# Finding Shipment Information (Destination) # Finding Shipment Information (Destination)
shipTo = li[-1].find('div', {'title': "List of countries is scrollable"}).text shipTo = temp[1].split("for")
shipTo = shipTo.replace("Ships to: ", "") shipTo = shipTo[0].strip()
shipTo = shipTo.strip()
if "certain countries" in shipTo:
countries = ""
tags = li[-1].find_all('span', {'class': "tag"})
for tag in tags:
country = tag.text.strip()
countries += country + ", "
shipTo = countries.strip(", ")
# Finding the Product description # Finding the Product description
describe = soup.find('div', {'class': "pre-line"}).text describe = soup.find('p', {'class': "card-text"}).text
describe = describe.replace("\n", " ") describe = describe.replace("\n", " ")
describe = describe.strip() describe = describe.strip()
@ -143,7 +134,7 @@ def darkfox_description_parser(soup):
# stores info it needs in different lists, these lists are returned after being organized # stores info it needs in different lists, these lists are returned after being organized
# @param: soup object looking at html page of listing page # @param: soup object looking at html page of listing page
# return: 'row' that contains a variety of lists that each hold info on the listing page # return: 'row' that contains a variety of lists that each hold info on the listing page
def darkfox_listing_parser(soup): def metaversemarket_listing_parser(soup):
# Fields to be parsed # Fields to be parsed
nm = 0 # Total_Products (Should be Integer) nm = 0 # Total_Products (Should be Integer)
mktName = "DarkFox" # 0 Marketplace_Name mktName = "DarkFox" # 0 Marketplace_Name
@ -169,7 +160,7 @@ def darkfox_listing_parser(soup):
success = [] # 20 Vendor_Successful_Transactions success = [] # 20 Vendor_Successful_Transactions
href = [] # 23 Product_Links (Urls) href = [] # 23 Product_Links (Urls)
listing = soup.findAll('div', {"class": "card"}) listing = soup.findAll('div', {"class": "col-12 col-sm-4 col-xl-3 product_item_col p-1"})
# Populating the Number of Products # Populating the Number of Products
nm = len(listing) nm = len(listing)
@ -183,58 +174,77 @@ def darkfox_listing_parser(soup):
href.append(link) href.append(link)
# Finding the Product # Finding the Product
product = bae[1].find('p').text product = bae[1].find('span', {"class": "text-primary"}).text
product = product.replace('\n', ' ') product = product.replace('\n', ' ')
product = product.replace(",", "") product = product.replace(",", "")
product = product.replace("...", "") product = product.replace("...", "")
product = product.strip() product = product.strip()
name.append(product) name.append(product)
bae = a.find('div', {'class': "media-content"}).find('div').find_all('div') # Finding Prices
price = a.find('strong').text
if len(bae) >= 5: price = price.replace("Buy for $", "")
# Finding Prices price = price.strip()
price = bae[0].text USD.append(price)
ud = price.replace(" USD", " ") # Finding the Vendor
# u = ud.replace("$","") temp = a.find('div', {'class': "mt-1 fs-12px"})
u = ud.replace(",", "") temp = temp.findAll('span')
u = u.strip() temp = temp[1].find('b').text
USD.append(u) vendor_name = temp.replace("@", "").strip()
# bc = (prc[1]).strip(' BTC') vendor.append(vendor_name)
# BTC.append(bc) # Finding the Category
cat = a.select_one('div[class="fs-12px"]')
# Finding the Vendor cat = cat.findAll('span')[1].text
vendor_name = bae[1].find('a').text cat = cat.text
vendor_name = vendor_name.replace(",", "") cat = cat.strip()
vendor_name = vendor_name.strip() category.append(cat)
vendor.append(vendor_name) badge = a.findAll('span', {'class': "badge bg-success"})
# Finding Number Sold and Quantity Left
# Finding the Category temp = badge[1].text
cat = bae[2].find('small').text temp = temp.split("/")
cat = cat.replace("Category: ", "") num = temp[0]
cat = cat.replace(",", "") num = num.strip()
cat = cat.strip() sold.append(num)
category.append(cat) quant = temp[1]
quant = quant.strip()
# Finding Number Sold and Quantity Left qLeft.append(quant)
num = bae[3].text # Finding Successful Transactions
num = num.replace("Sold: ", "") # NA
num = num.strip() # Finding Product review
sold.append(num) review = a.find('span', {'class': "badge bg-success fs-12px"}).text
review = review.replace("+ ", "")
quant = bae[4].find('small').text reviews.append(review)
quant = quant.replace("In stock: ", "") # Finding Descrption
quant = quant.strip() description = a.find('p', {'class': "alert alert-light text-ssbold p-1"}).text
qLeft.append(quant) description = description.replace("\n", " ")
description = description.strip()
# Finding Successful Transactions describe.append(description)
freq = bae[1].text # Finding Escrow
freq = freq.replace(vendor_name, "") es = a.find('span', {'class': "fw-bold"}).text.strip()
freq = re.sub(r'Vendor Level \d+', "", freq) escrow.append(es)
freq = freq.replace("(", "") # Finding Number of Views
freq = freq.replace(")", "") view = a.find('span', {'class': "badge bg-primary"}).text.strip()
freq = freq.strip() views.append(view)
success.append(freq) # Find where ships from
ships = a.find('div', {'class': "alert alert-info item_alert fs-12px p-1"})
ships = ships.findAll('b')
sFrom = ships[0].text.strips()
shipFrom.append(sFrom)
# Find where it ships to
sTo = ships[1].text.strips()
shipTo.append(sTo)
# Searching for CVE and MS categories # Searching for CVE and MS categories
cve = a.findAll(text=re.compile('CVE-\d{4}-\d{4}')) cve = a.findAll(text=re.compile('CVE-\d{4}-\d{4}'))
@ -275,7 +285,7 @@ def metaversemarket_links_parser(soup):
# Returning all links that should be visited by the Crawler # Returning all links that should be visited by the Crawler
href = [] href = []
listing = soup.findAll('div', {"class": "col-12 p-0"}) listing = soup.findAll('div', {"class": "col-12 col-sm-4 col-xl-3 product_item_col p-1"})
for a in listing: for a in listing:
bae = a.find('a', href=True) bae = a.find('a', href=True)


|||||||
x
 
000:0
Loading…
Cancel
Save