this is based on calsyslab project
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

133 lines
4.5 KiB

1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
1 year ago
  1. __author__ = 'DarkWeb'
  2. '''
  3. Starting point of the Darkweb Markets Mining
  4. '''
  5. from datetime import *
  6. from MarketPlaces.DarkFox.crawler_selenium import crawler as crawlerDarkFox
  7. from MarketPlaces.MikesGrandStore.crawler_selenium import crawler as crawlerMikesGrandStore
  8. from MarketPlaces.DarkTor.crawler_selenium import crawler as crawlerDarkTor
  9. from MarketPlaces.AnonymousMarketplace.crawler_selenium import crawler as crawlerAnonymousMarketplace
  10. from MarketPlaces.CityMarket.crawler_selenium import crawler as crawlerCityMarket
  11. from MarketPlaces.M00nkeyMarket.crawler_selenium import crawler as crawlerM00nkeyMarket
  12. from MarketPlaces.ViceCity.crawler_selenium import crawler as crawlerViceCity
  13. from MarketPlaces.CypherMarketplace.crawler_selenium import crawler as crawlerCypher
  14. from MarketPlaces.PabloEscobarMarket.crawler_selenium import crawler as crawlerPabloEscobar
  15. from MarketPlaces.DarkBazar.crawler_selenium import crawler as crawlerDarkBazar
  16. from MarketPlaces.Sonanza.crawler_selenium import crawler as crawlerSonanza
  17. from MarketPlaces.Kingdom.crawler_selenium import crawler as crawlerKingdom
  18. from MarketPlaces.BlackPyramid.crawler_selenium import crawler as crawlerBlackPyramid
  19. from MarketPlaces.Quest.crawler_selenium import crawler as crawlerQuest
  20. from MarketPlaces.Ares.crawler_selenium import crawler as crawlerAres
  21. from MarketPlaces.Bohemia.crawler_selenium import crawler as crawlerBohemia
  22. from MarketPlaces.TheDarkMarket.crawler_selenium import crawler as crawlerTheDarkMarket
  23. import configparser
  24. import os
  25. import subprocess
  26. config = configparser.ConfigParser()
  27. config.read('../../setup.ini')
  28. CURRENT_DATE = str("%02d" % date.today().month) + str("%02d" % date.today().day) + str("%04d" % date.today().year)
  29. # reads list of marketplaces
  30. def getMarkets():
  31. mkts = []
  32. with open('MarketsList.txt') as f:
  33. mkts = f.readlines()
  34. return mkts
  35. # Creates needed directories for marketplace if doesn't exist
  36. def createDirectory(mkt):
  37. # Package should already be there, holding crawler and parser
  38. # pagesDir = '../' + mkt + '/HTML_Pages'
  39. pagesDir = os.path.join(config.get('Project', 'shared_folder'), "MarketPlaces/" + mkt + "/HTML_Pages")
  40. if not os.path.isdir(pagesDir):
  41. os.makedirs(pagesDir)
  42. currentDateDir = pagesDir + '/' + CURRENT_DATE
  43. if not os.path.isdir(currentDateDir):
  44. os.mkdir(currentDateDir)
  45. listingDir = currentDateDir + '/Listing'
  46. if not os.path.isdir(listingDir):
  47. os.mkdir(listingDir)
  48. listReadDir = listingDir + '/Read'
  49. if not os.path.isdir(listReadDir):
  50. os.mkdir(listReadDir)
  51. descriptionDir = currentDateDir + '/Description'
  52. if not os.path.isdir(descriptionDir):
  53. os.mkdir(descriptionDir)
  54. descReadDir = descriptionDir + '/Read'
  55. if not os.path.isdir(descReadDir):
  56. os.mkdir(descReadDir)
  57. # Opens Tor Browser
  58. def opentor():
  59. global pid
  60. print("Connecting Tor...")
  61. pro = subprocess.Popen(config.get('TOR', 'firefox_binary_path'))
  62. pid = pro.pid
  63. # time.sleep(7.5)
  64. input('Press ENTER when Tor is connected to continue')
  65. return
  66. if __name__ == '__main__':
  67. # opentor()
  68. mktsList = getMarkets()
  69. for mkt in mktsList:
  70. mkt = mkt.replace('\n','')
  71. print("\nCreating listing and description directories ... for " + mkt)
  72. createDirectory(mkt)
  73. print("Directories created.")
  74. if mkt == "DarkFox":
  75. # for base in json["DarkFox"]["base"]:
  76. # if crawlerDarkFox(base["url"], base["categories"]):
  77. # break
  78. crawlerDarkFox()
  79. elif mkt == "MikesGrandStore":
  80. crawlerMikesGrandStore()
  81. elif mkt == "DarkTor":
  82. crawlerDarkTor()
  83. elif mkt == "AnonymousMarketplace":
  84. crawlerAnonymousMarketplace()
  85. elif mkt == "CityMarket":
  86. crawlerCityMarket()
  87. elif mkt == "M00nkeyMarket":
  88. crawlerM00nkeyMarket()
  89. elif mkt == "ViceCity":
  90. crawlerViceCity()
  91. elif mkt == "CypherMarketplace":
  92. crawlerCypher()
  93. elif mkt == "PabloEscobarMarket":
  94. crawlerPabloEscobar()
  95. elif mkt == "DarkBazar":
  96. crawlerDarkBazar()
  97. elif mkt == "Sonanza":
  98. crawlerSonanza()
  99. elif mkt == "Kingdom":
  100. crawlerKingdom()
  101. elif mkt == "BlackPyramid":
  102. crawlerBlackPyramid()
  103. elif mkt == "Quest":
  104. crawlerQuest()
  105. elif mkt == "Ares":
  106. crawlerAres()
  107. elif mkt == "TheDarkMarket":
  108. crawlerTheDarkMarket()
  109. print("\nScraping process completed!")