add parser

MarketPlaces/MetaVerseMarket/parser.py

@@ -11,7 +11,7 @@ from bs4 import BeautifulSoup
 # stores info it needs in different lists, these lists are returned after being organized
 # @param: soup object looking at html page of description page
 # return: 'row' that contains a variety of lists that each hold info on the description page
-def darkfox_description_parser(soup):
+def metaversemarket_description_parser(soup):
     # Fields to be parsed
 
     name = "-1"  # 0 Product_Name
@@ -36,67 +36,58 @@ def darkfox_description_parser(soup):
     EURO = "-1"  # 22 Product_EURO_SellingPrice
 
     # Finding Product Name
-    name = soup.find('h1').text
+    name = soup.find('div', {'class': "panel-heading"}).text.strip
-    name = name.replace('\n', ' ')
-    name = name.replace(",", "")
-    name = name.strip()
 
     # Finding Vendor
-    vendor = soup.find('h3').find('a').text.strip()
+    temp = soup.findAll('div', {'class': "col-xs-12 col-sm-6 mt-5"})
+    temp = temp[1].findAll('span')
+    temp = temp[1].find('b').text
+    name = temp.replace("@", "")
 
-    # Finding Vendor Rating
+    # Finding Product Reviews
-    rating = soup.find('span', {'class': "tag is-dark"}).text.strip()
+    review = soup.find('span', {'class': "badge bg-success fs-12px"}).text.strip()
 
     # Finding Successful Transactions
-    success = soup.find('h3').text
+    # NA
-    success = success.replace("Vendor: ", "")
-    success = success.replace(vendor, "")
-    success = success.replace("(", "")
-    success = success.replace(")", "")
-    success = success.strip()
 
-    bae = soup.find('div', {'class': "box"}).find_all('ul')
 
     # Finding Prices
-    USD = bae[1].find('strong').text.strip()
+    USD = soup.find('h3', {'class': "mb-2"}).text()
-
+    USD = USD.replace("Price: $", "").strip()
-    li = bae[2].find_all('li')
 
     # Finding Escrow
-    escrow = li[0].find('span', {'class': "tag is-dark"}).text.strip()
+    escrow = soup.find('div', {'class': "alert alert-info text-center fw-bold"}).text
+    escrow = escrow.replace('You are protected by ', "").strip()
 
     # Finding the Product Category
-    category = li[1].find('span', {'class': "tag is-dark"}).text.strip()
+    temp = soup.select('div[class="mt-2"]')
+    temp = temp[0].findAll('span')
+    category = temp[1].text.strip()
 
     # Finding the Product Quantity Available
-    left = li[3].find('span', {'class': "tag is-dark"}).text.strip()
+    # temp = soup.find('em', {'class': "icon ni ni-layers-fill"}).parent.parent.parent
+    # left = temp.text
+    # left = left.replace("Supply:", "")
+    # left = left.strip()
+    temp = soup.findAll('span', {'class': "badge bg-success"})
+    temp = temp[1].text.split("/")
+    left = temp[1].strip()
 
     # Finding Number Sold
-    sold = li[4].find('span', {'class': "tag is-dark"}).text.strip()
+    sold = temp[0].strip()
 
-    li = bae[3].find_all('li')
 
     # Finding Shipment Information (Origin)
-    if "Ships from:" in li[-2].text:
+    temp = soup.findAll('div', {'class': "alert alert-info"})
-        shipFrom = li[-2].text
+    temp = temp[1].text.split("to")
-        shipFrom = shipFrom.replace("Ships from: ", "")
+    shipFrom = temp[0].replace("Shipping from ", "").strip()
-        # shipFrom = shipFrom.replace(",", "")
-        shipFrom = shipFrom.strip()
 
     # Finding Shipment Information (Destination)
-    shipTo = li[-1].find('div', {'title': "List of countries is scrollable"}).text
+    shipTo = temp[1].split("for")
-    shipTo = shipTo.replace("Ships to: ", "")
+    shipTo = shipTo[0].strip()
-    shipTo = shipTo.strip()
-    if "certain countries" in shipTo:
-        countries = ""
-        tags = li[-1].find_all('span', {'class': "tag"})
-        for tag in tags:
-            country = tag.text.strip()
-            countries += country + ", "
-        shipTo = countries.strip(", ")
 
     # Finding the Product description
-    describe = soup.find('div', {'class': "pre-line"}).text
+    describe = soup.find('p', {'class': "card-text"}).text
     describe = describe.replace("\n", " ")
     describe = describe.strip()
 
@@ -143,7 +134,7 @@ def darkfox_description_parser(soup):
 # stores info it needs in different lists, these lists are returned after being organized
 # @param: soup object looking at html page of listing page
 # return: 'row' that contains a variety of lists that each hold info on the listing page
-def darkfox_listing_parser(soup):
+def metaversemarket_listing_parser(soup):
     # Fields to be parsed
     nm = 0  # Total_Products (Should be Integer)
     mktName = "DarkFox"  # 0 Marketplace_Name
@@ -169,7 +160,7 @@ def darkfox_listing_parser(soup):
     success = []  # 20 Vendor_Successful_Transactions
     href = []  # 23 Product_Links (Urls)
 
-    listing = soup.findAll('div', {"class": "card"})
+    listing = soup.findAll('div', {"class": "col-12 col-sm-4 col-xl-3 product_item_col p-1"})
 
     # Populating the Number of Products
     nm = len(listing)
@@ -183,58 +174,77 @@ def darkfox_listing_parser(soup):
         href.append(link)
 
         # Finding the Product
-        product = bae[1].find('p').text
+        product = bae[1].find('span', {"class": "text-primary"}).text
         product = product.replace('\n', ' ')
         product = product.replace(",", "")
         product = product.replace("...", "")
         product = product.strip()
         name.append(product)
 
-        bae = a.find('div', {'class': "media-content"}).find('div').find_all('div')
+        # Finding Prices
-
+        price = a.find('strong').text
-        if len(bae) >= 5:
+        price = price.replace("Buy for $", "")
-            # Finding Prices
+        price = price.strip()
-            price = bae[0].text
+        USD.append(price)
-            ud = price.replace(" USD", " ")
+        # Finding the Vendor
-            # u = ud.replace("$","")
+        temp = a.find('div', {'class': "mt-1 fs-12px"})
-            u = ud.replace(",", "")
+        temp = temp.findAll('span')
-            u = u.strip()
+        temp = temp[1].find('b').text
-            USD.append(u)
+        vendor_name = temp.replace("@", "").strip()
-            # bc = (prc[1]).strip(' BTC')
+        vendor.append(vendor_name)
-            # BTC.append(bc)
+        # Finding the Category
-
+        cat = a.select_one('div[class="fs-12px"]')
-            # Finding the Vendor
+        cat = cat.findAll('span')[1].text
-            vendor_name = bae[1].find('a').text
+        cat = cat.text
-            vendor_name = vendor_name.replace(",", "")
+        cat = cat.strip()
-            vendor_name = vendor_name.strip()
+        category.append(cat)
-            vendor.append(vendor_name)
+        badge = a.findAll('span', {'class': "badge bg-success"})
-
+        # Finding Number Sold and Quantity Left
-            # Finding the Category
+        temp = badge[1].text
-            cat = bae[2].find('small').text
+        temp = temp.split("/")
-            cat = cat.replace("Category: ", "")
+        num = temp[0]
-            cat = cat.replace(",", "")
+        num = num.strip()
-            cat = cat.strip()
+        sold.append(num)
-            category.append(cat)
+        quant = temp[1]
-
+        quant = quant.strip()
-            # Finding Number Sold and Quantity Left
+        qLeft.append(quant)
-            num = bae[3].text
+        # Finding Successful Transactions
-            num = num.replace("Sold: ", "")
+        # NA
-            num = num.strip()
+        # Finding Product review
-            sold.append(num)
+        review = a.find('span', {'class': "badge bg-success fs-12px"}).text
-
+        review = review.replace("+ ", "")
-            quant = bae[4].find('small').text
+        reviews.append(review)
-            quant = quant.replace("In stock: ", "")
+        # Finding Descrption
-            quant = quant.strip()
+        description = a.find('p', {'class': "alert alert-light text-ssbold p-1"}).text
-            qLeft.append(quant)
+        description = description.replace("\n", " ")
-
+        description = description.strip()
-            # Finding Successful Transactions
+        describe.append(description)
-            freq = bae[1].text
+        # Finding Escrow
-            freq = freq.replace(vendor_name, "")
+        es = a.find('span', {'class': "fw-bold"}).text.strip()
-            freq = re.sub(r'Vendor Level \d+', "", freq)
+        escrow.append(es)
-            freq = freq.replace("(", "")
+        # Finding Number of Views
-            freq = freq.replace(")", "")
+        view = a.find('span', {'class': "badge bg-primary"}).text.strip()
-            freq = freq.strip()
+        views.append(view)
-            success.append(freq)
+        # Find where ships from
+        ships = a.find('div', {'class': "alert alert-info item_alert fs-12px p-1"})
+        ships = ships.findAll('b')
+        sFrom = ships[0].text.strips()
+        shipFrom.append(sFrom)
+
+        # Find where it ships to
+        sTo = ships[1].text.strips()
+        shipTo.append(sTo)
 
         # Searching for CVE and MS categories
         cve = a.findAll(text=re.compile('CVE-\d{4}-\d{4}'))
@@ -275,7 +285,7 @@ def metaversemarket_links_parser(soup):
     # Returning all links that should be visited by the Crawler
 
     href = []
-    listing = soup.findAll('div', {"class": "col-12 p-0"})
+    listing = soup.findAll('div', {"class": "col-12 col-sm-4 col-xl-3 product_item_col p-1"})
 
     for a in listing:
         bae = a.find('a', href=True)