You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
73 lines
2.0 KiB
73 lines
2.0 KiB
__author__ = 'DarkWeb'
|
|
|
|
'''
|
|
Starting point of the Darkweb Mining Platform
|
|
'''
|
|
|
|
import os
|
|
from datetime import *
|
|
from MarketPlaces.DarkFox.crawler_selenium import crawler as crawlerDarkFox
|
|
from MarketPlaces.Tor2door.crawler_selenium import crawler as crawlerTor2door
|
|
from MarketPlaces.ThiefWorld.crawler_selenium import crawler as crawlerThiefWorld
|
|
|
|
import time
|
|
|
|
CURRENT_DATE = str("%02d" % date.today().month) + str("%02d" % date.today().day) + str("%04d" % date.today().year)
|
|
|
|
|
|
# reads list of marketplaces
|
|
def getMarkets():
|
|
mkts = []
|
|
with open('MarketsList.txt') as f:
|
|
mkts = f.readlines()
|
|
return mkts
|
|
|
|
|
|
# Creates needed directories for marketplace if doesn't exist
|
|
def createDirectory(mkt):
|
|
|
|
# Package should already be there, holding crawler and parser
|
|
pagesDir = '../' + mkt + '/HTML_Pages'
|
|
if not os.path.isdir(pagesDir):
|
|
os.makedirs(pagesDir)
|
|
|
|
currentDateDir = pagesDir + '/' + CURRENT_DATE
|
|
if not os.path.isdir(currentDateDir):
|
|
os.mkdir(currentDateDir)
|
|
|
|
listingDir = currentDateDir + '/Listing'
|
|
if not os.path.isdir(listingDir):
|
|
os.mkdir(listingDir)
|
|
|
|
listReadDir = listingDir + '/Read'
|
|
if not os.path.isdir(listReadDir):
|
|
os.mkdir(listReadDir)
|
|
|
|
descriptionDir = currentDateDir + '/Description'
|
|
if not os.path.isdir(descriptionDir):
|
|
os.mkdir(descriptionDir)
|
|
|
|
descReadDir = descriptionDir + '/Read'
|
|
if not os.path.isdir(descReadDir):
|
|
os.mkdir(descReadDir)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
mktsList = getMarkets()
|
|
|
|
for mkt in mktsList:
|
|
mkt = mkt.replace('\n','')
|
|
|
|
print("Creating listing and description directories ...")
|
|
createDirectory(mkt)
|
|
time.sleep(5)
|
|
input("Directories created successfully. Press ENTER to continue\n")
|
|
|
|
if mkt == "DarkFox":
|
|
crawlerDarkFox()
|
|
elif mkt == 'Tor2door':
|
|
crawlerTor2door()
|
|
elif mkt == "ThiefWorld":
|
|
crawlerThiefWorld()
|
|
|
|
print("Scraping process completed successfully!")
|