You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
153 lines
5.6 KiB
153 lines
5.6 KiB
__author__ = 'DarkWeb'
|
|
|
|
'''
|
|
Starting point of the Darkweb Markets Mining
|
|
'''
|
|
|
|
from datetime import *
|
|
from MarketPlaces.DarkFox.crawler_selenium import crawler as crawlerDarkFox
|
|
from MarketPlaces.Tor2door.crawler_selenium import crawler as crawlerTor2door
|
|
from MarketPlaces.ThiefWorld.crawler_selenium import crawler as crawlerThiefWorld
|
|
from MarketPlaces.TorBay.crawler_selenium import crawler as crawlerTorBay
|
|
from MarketPlaces.LionMarketplace.crawler_selenium import crawler as crawlerLionMarketplace
|
|
from MarketPlaces.TorMarket.crawler_selenium import crawler as crawlerTorMarket
|
|
from MarketPlaces.MikesGrandStore.crawler_selenium import crawler as crawlerMikesGrandStore
|
|
from MarketPlaces.DarkTor.crawler_selenium import crawler as crawlerDarkTor
|
|
from MarketPlaces.DigitalThriftShop.crawler_selenium import crawler as crawlerDigitalThriftShop
|
|
from MarketPlaces.AnonymousMarketplace.crawler_selenium import crawler as crawlerAnonymousMarketplace
|
|
from MarketPlaces.Apocalypse.crawler_selenium import crawler as crawlerApocalypseMarketplace
|
|
from MarketPlaces.CityMarket.crawler_selenium import crawler as crawlerCityMarket
|
|
from MarketPlaces.DarkMatter.crawler_selenium import crawler as crawlerDarkMatter
|
|
from MarketPlaces.M00nkeyMarket.crawler_selenium import crawler as crawlerM00nkeyMarket
|
|
from MarketPlaces.ViceCity.crawler_selenium import crawler as crawlerViceCity
|
|
from MarketPlaces.HiddenMarket.crawler_selenium import crawler as crawlerHiddenMarket
|
|
from MarketPlaces.RobinhoodMarket.crawler_selenium import crawler as crawlerRobinhoodMarket
|
|
from MarketPlaces.Nexus.crawler_selenium import crawler as crawlerNexus
|
|
from MarketPlaces.CypherMarketplace.crawler_selenium import crawler as crawlerCypher
|
|
from MarketPlaces.DarkBazar.crawler_selenium import crawler as crawlerDarkBazar
|
|
from MarketPlaces.PabloEscobarMarket.crawler_selenium import crawler as crawlerPabloEscobar
|
|
from MarketPlaces.AnonMarket.crawler_selenium import crawler as crawlerAnonMarket
|
|
from MarketPlaces.MetaVerseMarket.crawler_selenium import crawler as crawlerMetaVerse
|
|
|
|
import configparser
|
|
import os
|
|
import subprocess
|
|
|
|
config = configparser.ConfigParser()
|
|
config.read('../../setup.ini')
|
|
CURRENT_DATE = str("%02d" % date.today().month) + str("%02d" % date.today().day) + str("%04d" % date.today().year)
|
|
|
|
|
|
# reads list of marketplaces
|
|
def getMarkets():
|
|
mkts = []
|
|
with open('MarketsList.txt') as f:
|
|
mkts = f.readlines()
|
|
return mkts
|
|
|
|
|
|
# Creates needed directories for marketplace if doesn't exist
|
|
def createDirectory(mkt):
|
|
|
|
# Package should already be there, holding crawler and parser
|
|
# pagesDir = '../' + mkt + '/HTML_Pages'
|
|
pagesDir = os.path.join(config.get('Project', 'shared_folder'), "MarketPlaces/" + mkt + "/HTML_Pages")
|
|
if not os.path.isdir(pagesDir):
|
|
os.makedirs(pagesDir)
|
|
|
|
currentDateDir = pagesDir + '/' + CURRENT_DATE
|
|
if not os.path.isdir(currentDateDir):
|
|
os.mkdir(currentDateDir)
|
|
|
|
listingDir = currentDateDir + '/Listing'
|
|
if not os.path.isdir(listingDir):
|
|
os.mkdir(listingDir)
|
|
|
|
listReadDir = listingDir + '/Read'
|
|
if not os.path.isdir(listReadDir):
|
|
os.mkdir(listReadDir)
|
|
|
|
descriptionDir = currentDateDir + '/Description'
|
|
if not os.path.isdir(descriptionDir):
|
|
os.mkdir(descriptionDir)
|
|
|
|
descReadDir = descriptionDir + '/Read'
|
|
if not os.path.isdir(descReadDir):
|
|
os.mkdir(descReadDir)
|
|
|
|
|
|
# Opens Tor Browser
|
|
def opentor():
|
|
global pid
|
|
print("Connecting Tor...")
|
|
pro = subprocess.Popen(config.get('TOR', 'firefox_binary_path'))
|
|
pid = pro.pid
|
|
# time.sleep(7.5)
|
|
input('Press ENTER when Tor is connected to continue')
|
|
return
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
opentor()
|
|
|
|
mktsList = getMarkets()
|
|
|
|
for mkt in mktsList:
|
|
mkt = mkt.replace('\n','')
|
|
|
|
print("\nCreating listing and description directories ... for " + mkt)
|
|
createDirectory(mkt)
|
|
print("Directories created.")
|
|
|
|
if mkt == "DarkFox":
|
|
# for base in json["DarkFox"]["base"]:
|
|
# if crawlerDarkFox(base["url"], base["categories"]):
|
|
# break
|
|
crawlerDarkFox()
|
|
elif mkt == 'Tor2door':
|
|
crawlerTor2door()
|
|
elif mkt == "ThiefWorld":
|
|
crawlerThiefWorld()
|
|
elif mkt == "TorBay":
|
|
crawlerTorBay()
|
|
elif mkt == "LionMarketplace":
|
|
crawlerLionMarketplace()
|
|
elif mkt == "TorMarket":
|
|
crawlerTorMarket()
|
|
elif mkt == "MikesGrandStore":
|
|
crawlerMikesGrandStore()
|
|
elif mkt == "DarkTor":
|
|
crawlerDarkTor()
|
|
elif mkt == "DigitalThriftShop":
|
|
crawlerDigitalThriftShop()
|
|
elif mkt == "AnonymousMarketplace":
|
|
crawlerAnonymousMarketplace()
|
|
elif mkt == "Apocalypse":
|
|
crawlerApocalypseMarketplace()
|
|
elif mkt == "CityMarket":
|
|
crawlerCityMarket()
|
|
elif mkt == "DarkMatter":
|
|
crawlerDarkMatter()
|
|
elif mkt == "M00nkeyMarket":
|
|
crawlerM00nkeyMarket()
|
|
elif mkt == "ViceCity":
|
|
crawlerViceCity()
|
|
elif mkt == "HiddenMarket":
|
|
crawlerHiddenMarket()
|
|
elif mkt == "RobinhoodMarket":
|
|
crawlerRobinhoodMarket()
|
|
elif mkt == "Nexus":
|
|
crawlerNexus()
|
|
elif mkt == "CypherMarketplace":
|
|
crawlerCypher()
|
|
elif mkt == "DarkBazar":
|
|
crawlerDarkBazar()
|
|
elif mkt == "PabloEscobarMarket":
|
|
crawlerPabloEscobar()
|
|
elif mkt == "AnonMarket":
|
|
crawlerAnonMarket()
|
|
elif mkt == "MetaVerseMarket":
|
|
crawlerMetaVerse()
|
|
|
|
print("\nScraping process completed!")
|