Dark Market Crawler and Parser

MarketPlaces/Initialization/markets_mining.py

@@ -25,6 +25,7 @@ from MarketPlaces.TheDarkMarket.crawler_selenium import crawler as crawlerTheDar
 from MarketPlaces.GoFish.crawler_selenium import crawler as crawlerGoFish
 from MarketPlaces.ZeroDay.crawler_selenium import crawler as crawlerZeroDay
 from MarketPlaces.Torzon.crawler_selenium import crawler as crawlerTorzon
+from MarketPlaces.DarkMarket.crawler_selenium import crawler as crawlerDarkMarket
 
 import configparser
 import os
@@ -138,5 +139,7 @@ if __name__ == '__main__':
             crawlerZeroDay()
         elif mkt == "Torzon":
             crawlerTorzon()
+        elif mkt == "DarkMarket":
+            crawlerDarkMarket()
 
     print("\nScraping process completed!")

MarketPlaces/Initialization/prepare_parser.py

@@ -27,6 +27,7 @@ from MarketPlaces.WeTheNorth.parser import *
 from MarketPlaces.Torzon.parser import *
 from MarketPlaces.GoFish.parser import *
 from MarketPlaces.ZeroDay.parser import *
+from MarketPlaces.DarkMarket.parser import *
 
 from MarketPlaces.Classifier.classify_product import predict
 from Translator.translate import translate
@@ -158,7 +159,7 @@ def parse_listing(marketPlace, listingFile, soup, createLog, logFile):
         elif marketPlace == "CypherMarketplace":
             rw = cyphermarketplace_listing_parser(soup)
         elif marketPlace == "TheDarkMarket":
-            rw = darkmarket_listing_parser(soup)
+            rw = thedarkmarket_listing_parser(soup)
         elif marketPlace == "WeTheNorth":
             rw = wethenorth_listing_parser(soup)
         elif marketPlace == "GoFish":
@@ -167,6 +168,8 @@ def parse_listing(marketPlace, listingFile, soup, createLog, logFile):
             rw = zeroday_listing_parser(soup)
         elif marketPlace == "Torzon":
             rw = torzon_listing_parser(soup)
+        elif marketPlace == "DarkMarket":
+            rw = darkmarket_listing_parser(soup)
         else:
             print("MISSING CALL TO LISTING PARSER IN PREPARE_PARSER.PY!")
             raise Exception
@@ -216,7 +219,7 @@ def parse_description(marketPlace, descriptionFile, soup, createLog, logFile):
         elif marketPlace == "CypherMarketplace":
             rmm = cyphermarketplace_description_parser(soup)
         elif marketPlace == "TheDarkMarket":
-            rmm = darkmarket_description_parser(soup)
+            rmm = thedarkmarket_description_parser(soup)
         elif marketPlace == "WeTheNorth":
             rmm = wethenorth_description_parser(soup)
         elif marketPlace == "GoFish":
@@ -225,6 +228,8 @@ def parse_description(marketPlace, descriptionFile, soup, createLog, logFile):
             rmm = zeroday_description_parser(soup)
         elif marketPlace == "Torzon":
             rmm = torzon_description_parser(soup)
+        elif marketPlace == "DarkMarket":
+            rmm = darkmarket_description_parser(soup)
         else:
             print("MISSING CALL TO DESCRIPTION PARSER IN PREPARE_PARSER.PY!")
             raise Exception
@@ -250,6 +255,10 @@ def get_source_language(marketPlace):
         lang = 'english'
     elif marketPlace == "Incogsnoo":
         lang = 'english'
+    elif marketPlace == "CityMarket":
+        lang = 'english'
+    elif marketPlace == "DarkMarket":
+        lang = 'english'
     else:
         print("MISSING CALL TO GET LANGUAGE IN PREPARE_PARSER.PY!")
         lang = 'auto'

MarketPlaces/TheDarkMarket/crawler_selenium.py

@@ -22,7 +22,7 @@ from datetime import date
 import subprocess
 from bs4 import BeautifulSoup
 from MarketPlaces.Initialization.prepare_parser import new_parse
-from MarketPlaces.TheDarkMarket.parser import darkmarket_links_parser
+from MarketPlaces.TheDarkMarket.parser import thedarkmarket_links_parser
 from MarketPlaces.Utilities.utilities import cleanHTML
 
 counter = 1

MarketPlaces/TheDarkMarket/parser.py

@@ -8,7 +8,7 @@ from bs4 import BeautifulSoup, ResultSet, Tag
 
 
 # This is the method to parse the Description Pages (one page to each Product in the Listing Pages)
-def darkmarket_description_parser(soup: BeautifulSoup):
+def thedarkmarket_description_parser(soup: BeautifulSoup):
 
     # Fields to be parsed
 
@@ -75,7 +75,7 @@ def darkmarket_description_parser(soup: BeautifulSoup):
 
 
 # This is the method to parse the Listing Pages
-def darkmarket_listing_parser(soup: BeautifulSoup):
+def thedarkmarket_listing_parser(soup: BeautifulSoup):
 
     # Fields to be parsed
     nm = 0                                    # *Total_Products (Should be Integer)
@@ -165,7 +165,7 @@ def darkmarket_listing_parser(soup: BeautifulSoup):
                             image_vendor)
 
 
-def darkmarket_links_parser(soup: BeautifulSoup):
+def thedarkmarket_links_parser(soup: BeautifulSoup):
 
     # Returning all links that should be visited by the Crawler
     href = []