this is based on calsyslab project
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

194 lines
8.1 KiB

  1. __author__ = 'Helium'
  2. # Here, we are importing the auxiliary functions to clean or convert data
  3. from MarketPlaces.Utilities.utilities import *
  4. # Here, we are importing BeautifulSoup to search through the HTML tree
  5. from bs4 import BeautifulSoup
  6. # This is the method to parse the Description Pages (one page to each Product in the Listing Pages)
  7. def kingdom_description_parser(soup):
  8. # Fields to be parsed
  9. vendor = "-1" # 0 *Vendor_Name
  10. success = "-1" # 1 Vendor_Successful_Transactions
  11. rating_vendor = "-1" # 2 Vendor_Rating
  12. name = "-1" # 3 *Product_Name
  13. describe = "-1" # 4 Product_Description
  14. CVE = "-1" # 5 Product_CVE_Classification (Common Vulnerabilities and Exposures)
  15. MS = "-1" # 6 Product_MS_Classification (Microsoft Security)
  16. category = "-1" # 7 Product_Category
  17. views = "-1" # 8 Product_Number_Of_Views
  18. reviews = "-1" # 9 Product_Number_Of_Reviews
  19. rating_item = "-1" # 10 Product_Rating
  20. addDate = "-1" # 11 Product_AddedDate
  21. BTC = "-1" # 12 Product_BTC_SellingPrice
  22. USD = "-1" # 13 Product_USD_SellingPrice
  23. EURO = "-1" # 14 Product_EURO_SellingPrice
  24. sold = "-1" # 15 Product_QuantitySold
  25. left = "-1" # 16 Product_QuantityLeft
  26. shipFrom = "-1" # 17 Product_ShippedFrom
  27. shipTo = "-1" # 18 Product_ShippedTo
  28. image = "-1" # 19 Product_Image
  29. vendor_image = "-1" # 20 Vendor_Image
  30. # Finding Product Name
  31. tag = soup.find('div', {"class": "col-md-9"})
  32. desc = tag.find('div',{"class": "col-md-8"}).find('div', {"class": "box-cont"})
  33. name = tag.find('div',{"class": "col-md-8"}).find('div', {"class": "box-head"}).text
  34. name = cleanString(name).strip()
  35. # Finding Prices
  36. # Kingdom prices can be shown in a variety of currencies, not all in USD, so keeping currency
  37. rows = desc.find_all('div', {"class", "row"}, recursive=False)
  38. USD = rows[-1].find('div', {"class": "row"}).find('h3').text
  39. USD = cleanNumbers(USD).strip()
  40. BTC = rows[-1].find('div', {"class": "row"}).find_next_sibling('div').find('span').text
  41. BTC = cleanNumbers(BTC).strip()
  42. # Finding Vendor
  43. vendor = rows[0].select_one('a[href^="/user"]').text
  44. vendor = cleanString(vendor).strip()
  45. # Finding Shipment Information (Origem)
  46. descs = rows[0].find_all('div', {"class": "col-md-3 text-right"})
  47. shipFrom = descs[2].text
  48. shipFrom = cleanString(shipFrom).strip()
  49. # Finding Shipment Information (Destiny)
  50. shipTo = rows[-1].find('div', {"class": "col-md-6"}).text
  51. shipTo = shipTo.replace("Ship to:","")
  52. shipTo = cleanString(shipTo).strip()
  53. if shipTo == '':
  54. shipTo = "-1"
  55. # Finding the Product Category
  56. category = descs[0].text
  57. category = cleanString(category).strip()
  58. # Finding the Product Quantity Available
  59. left = descs[1].text
  60. left = cleanString(left).strip()
  61. # Finding when the Product was Added
  62. dt = descs[-1].text.strip()
  63. addDate = datetime.strptime(dt, '%d.%m.%Y')
  64. # Finding the Product description
  65. describe = cleanString(soup.find('div', {"id": "descriptionContent"}).text).strip()
  66. # Finding the Number of Product Reviews
  67. reviews = str(len(soup.find('div', {"id": "feedbackContent"}).find_all(recursive=False)))
  68. # Searching for CVE and MS categories
  69. # no cve or ms in Kingdom
  70. # Populating the final variable (this should be a list with all fields scraped)
  71. row = (vendor, rating_vendor, success, name, describe, CVE, MS, category, views, reviews, rating_item, addDate,
  72. BTC, USD, EURO, sold, left, shipFrom, shipTo, image, vendor_image)
  73. # Sending the results
  74. return row
  75. def kingdom_listing_parser(soup):
  76. # Fields to be parsed
  77. nm = 0 # *Total_Products (Should be Integer)
  78. mktName = "Kingdom" # 0 *Marketplace_Name
  79. vendor = [] # 1 *Vendor y
  80. rating_vendor = [] # 2 Vendor_Rating
  81. success = [] # 3 Vendor_Successful_Transactions
  82. name = [] # 4 *Product_Name y
  83. CVE = [] # 5 Product_CVE_Classification (Common Vulnerabilities and Exposures)
  84. MS = [] # 6 Product_MS_Classification (Microsoft Security)
  85. category = [] # 7 Product_Category y
  86. describe = [] # 8 Product_Description
  87. views = [] # 9 Product_Number_Of_Views
  88. reviews = [] # 10 Product_Number_Of_Reviews
  89. rating_item = [] # 11 Product_Rating
  90. addDate = [] # 12 Product_AddDate
  91. BTC = [] # 13 Product_BTC_SellingPrice
  92. USD = [] # 14 Product_USD_SellingPrice y
  93. EURO = [] # 15 Product_EURO_SellingPrice
  94. sold = [] # 16 Product_QuantitySold
  95. qLeft =[] # 17 Product_QuantityLeft
  96. shipFrom = [] # 18 Product_ShippedFrom
  97. shipTo = [] # 19 Product_ShippedTo
  98. image = [] # 20 Product_Image
  99. image_vendor = [] # 21 Vendor_Image
  100. href = [] # 22 Product_Links
  101. listing = soup.find('div', {"id": "p0"}).find('div').find_all('div', {"class": "row"}, recursive=False)
  102. # Populating the Number of Products
  103. nm = len(listing)
  104. for a in listing:
  105. # Finding Prices
  106. #in array USD, there may be prices not in USD, so includes currency as well
  107. prices = a.find('div', {"class": "col-md-3"})
  108. u = prices.find('h3').text
  109. USD.append(cleanNumbers(u).strip())
  110. bc = prices.find('div').find('span').text
  111. BTC.append(cleanNumbers(bc).strip())
  112. # Finding the Product
  113. product = a.find('div', {"class": "col-md-7"}).select_one('a[href^="/offer/view?"]').text
  114. name.append(cleanString(product).strip())
  115. # Finding Product Image
  116. product_image = a.find('img')
  117. product_image = product_image.get('src')
  118. product_image = product_image.split('base64,')[-1]
  119. image.append(product_image)
  120. # Finding the Vendor
  121. vendor_name = a.select_one('a[href^="/user"]').text
  122. vendor_name = vendor_name.replace('/', '')
  123. vendor.append(cleanString(vendor_name).strip())
  124. # Finding Views
  125. product_views = a.find('div', {"class": "col-md-7"}).find_all('p')[0].text
  126. views.append(cleanNumbers(product_views).strip())
  127. # Finding Sold
  128. product_sold = a.find('div', {"class": "base-label label label-rounded label-success"})
  129. if product_sold is not None:
  130. sold.append(cleanNumbers(product_sold.text).strip())
  131. else:
  132. sold.append("-1")
  133. # Adding the url to the list of urls
  134. link = a.find('div', {"class": "col-md-7"}).select_one('a[href^="/offer/view?"]')['href']
  135. href.append(link)
  136. # Searching for CVE and MS categories
  137. # cve and ms not in kingdom
  138. # Populate the final variable (this should be a list with all fields scraped)
  139. return organizeProducts(mktName, nm, vendor, rating_vendor, success, name, CVE, MS, category, describe, views,
  140. reviews, rating_item, addDate, BTC, USD, EURO, sold, qLeft, shipFrom, shipTo, href,
  141. image, image_vendor)
  142. def kingdom_links_parser(soup):
  143. # Returning all links that should be visited by the Crawler
  144. href = []
  145. listing = soup.findAll('div', {"class": "col-md-7"})
  146. for a in listing:
  147. link = a.select_one('a[href^="/offer/view?"]')
  148. link = link['href']
  149. href.append(link)
  150. return href