__author__ = 'Helium'
import glob
import os
import codecs
import shutil
from MarketPlaces.DB_Connection.db_connection import *
from MarketPlaces.DarkFox.parser import *
from MarketPlaces.Tor2door.parser import *
from MarketPlaces.Apocalypse.parser import *
from MarketPlaces.ThiefWorld.parser import *
from MarketPlaces.AnonymousMarketplace.parser import *
from MarketPlaces.Classifier.classify_product import predict
def mergePages(rmm, rec):
# key = u"Pr:" + rec[1].upper() + u" Vendor:" + rec[18].upper()
# key = rec[23]
print("----------------- Matched: " + rec[1] + "--------------------")
if rec[1] == "-1": # name_vendor
rec[1] = rmm[0]
if rec[2] == "-1": # rating_vendor
rec[2] = rmm[1]
if rec[3] == "-1": # success_vendor
rec[3] = rmm[2]
if rec[4] == "-1": # name_item
rec[4] = rmm[3]
if rec[5] == "-1": # description_item
rec[5] = rmm[4]
if rec[6] == "-1": # cve_item
rec[6] = rmm[5]
if rec[7] == "-1": # ms_item
rec[7] = rmm[6]
if rec[8] == "-1": # category_item
rec[8] = rmm[7]
if rec[9] == "-1": # views_item
rec[9] = rmm[8]
if rec[10] == "-1": # reviews_item
rec[10] = rmm[9]
if rec[11] == "-1": # rating_item
rec[11] = rmm[10]
if rec[12] == "-1": # adddate_item
rec[12] = rmm[11]
if rec[13] == "-1": # btc_item
rec[13] = rmm[12]
if rec[14] == "-1": # usd_item
rec[14] = rmm[13]
if rec[15] == "-1": # euro_item
rec[15] = rmm[14]
if rec[16] == "-1": # quantitysold_item
rec[16] = rmm[15]
if rec[17] == "-1": # quantityleft_item
rec[17] = rmm[16]
if rec[18] == "-1": # shippedfrom_item
rec[18] = rmm[17]
if rec[19] == "-1": # shippedto_item
rec[19] = rmm[18]
return rec
def persist_data(url, row, cur):
marketPlace = create_marketPlace(cur, row, url)
vendor = create_vendor(cur, row, marketPlace)
create_items(cur, row, marketPlace, vendor)
def new_parse(marketPlace, url, createLog):
from MarketPlaces.Initialization.markets_mining import config, CURRENT_DATE
print("Parsing the " + marketPlace + " marketplace and conduct data classification to store the information in the database.")
# ini = time.time()
# Connecting to the database
con = connectDataBase()
cur = con.cursor()
# Creating the tables (The database should be created manually)
create_database(cur, con)
nError = 0
lines = [] # listing pages
lns = [] # description pages
detPage = {}
#Creating the log file for each Market Place
if createLog:
if not os.path.exists("./" + marketPlace + "/Logs/" + marketPlace + "_" + CURRENT_DATE + ".log"):
logFile = open("./" + marketPlace + "/Logs/" + marketPlace + "_" + CURRENT_DATE + ".log", "w")
else:
print("Files of the date " + CURRENT_DATE + " from the Market Place " + marketPlace +
" were already read. Delete the referent information in the Data Base and also delete the log file"
" in the _Logs folder to read files from this Market Place of this date again.")
raise SystemExit
mainDir = os.path.join(config.get('Project', 'shared_folder'), "MarketPlaces/" + marketPlace + "/HTML_Pages")
# Reading the Listing Html Pages
for fileListing in glob.glob(os.path.join(mainDir, CURRENT_DATE + "\\Listing", '*.html')):
lines.append(fileListing)
# Reading the Description Html Pages
for fileDescription in glob.glob(os.path.join(mainDir, CURRENT_DATE + "\\Description", '*.html')):
lns.append(fileDescription)
# Parsing the Description Pages and put the tag's content into a dictionary (Hash table)
for index, line2 in enumerate(lns):
print("Reading description folder of '" + marketPlace + "', file '" + os.path.basename(line2) + "', index= " + str(index + 1) + " ... " + str(len(lns)))
try:
html = codecs.open(line2.strip('\n'), encoding='utf8')
soup = BeautifulSoup(html, "html.parser")
html.close()
except:
try:
html = open(line2.strip('\n'))
soup = BeautifulSoup(html, "html.parser")
html.close()
except:
nError += 1
print("There was a problem to read the file " + line2 + " in the Description section!")
if createLog:
logFile.write(str(nError) + ". There was a problem to read the file " + line2 + " in the Description section.\n")
continue
try:
if marketPlace == "DarkFox":
rmm = darkfox_description_parser(soup)
elif marketPlace == "Tor2door":
rmm = tor2door_description_parser(soup)
elif marketPlace == "Apocalypse":
rmm = apocalypse_description_parser(soup)
elif marketPlace == "ThiefWorld":
rmm = thiefWorld_description_parser(soup)
elif marketPlace =="AnonymousMarketplace":
rmm = anonymousMarketplace_description_parser(soup)
print(rmm)
# key = u"Pr:" + rmm[0].upper()[:desc_lim1] + u" Vendor:" + rmm[13].upper()[:desc_lim2]
key = u"Url:" + os.path.basename(line2).replace(".html", "")
# save file address with description record in memory
detPage[key] = {'rmm': rmm, 'filename': os.path.basename(line2)}
except :
nError += 1
print("There was a problem to parse the file " + line2 + " in the Description section!")
if createLog:
logFile.write(str(nError) + ". There was a problem to parse the file " + line2 + " in the Description section.\n")
# Parsing the Listing Pages and put the tag's content into a list
for index, line1 in enumerate(lines):
print("Reading listing folder of '" + marketPlace + "', file '" + os.path.basename(line1) + "', index= " + str(index + 1) + " ... " + str(len(lines)))
readError = False
try:
html = codecs.open(line1.strip('\n'), encoding='utf8')
soup = BeautifulSoup(html, "html.parser")
html.close()
except:
try:
html = open(line1.strip('\n'))
soup = BeautifulSoup(html, "html.parser")
html.close()
except:
nError += 1
print("There was a problem to read the file " + line1 + " in the Listing section!")
if createLog:
logFile.write(str(nError) + ". There was a problem to read the file " + line1 + " in the Listing section.\n")
readError = True
if not readError:
print("Hello!")
parseError = False
try:
if marketPlace == "DarkFox":
rw = darkfox_listing_parser(soup)
elif marketPlace == "Tor2door":
rw = tor2door_listing_parser(soup)
elif marketPlace == "Apocalypse":
rw = apocalypse_listing_parser(soup)
elif marketPlace == "ThiefWorld":
rw = thiefWorld_listing_parser(soup)
elif marketPlace == "AnonymousMarketplace":
rw = anonymousMarketplace_listing_parser(soup)
else:
parseError = True
except Exception as e:
raise e
nError += 1
print("There was a problem to parse the file " + line1 + " in the listing section!")
if createLog:
logFile.write(
str(nError) + ". There was a problem to parse the file " + line1 + " in the Listing section.\n")
parseError = True
if not parseError:
persistError = False
moveError = False
num_in_db = 0
num_persisted_moved = 0
for rec in rw:
rec = rec.split(',')
print(rec)
# if len(detPage) > 0: #It was created here just because Zeroday Market does not have Description Pages
# key = rec[23]
# key = u"Pr:" + rec[1].upper()[:list_lim1] + u" Vendor:" + rec[18].upper()[:list_lim2]
key = u"Url:" + cleanLink(rec[20])
print(key)
# if the associated description page is parsed
if key in detPage:
# rec = mergePages(detPage, rec)
# Combining the information from Listing and Description Pages
rmm = detPage[key]['rmm']
rec = mergePages(rmm, rec)
# Append to the list the classification of the product
# rec.append(str(predict(rec[1], rec[5], language='markets')))
rec.append(str(predict(rec[4], rec[5], language='sup_english')))
# Persisting the information in the database
try:
persist_data(url, tuple(rec), cur)
con.commit()
except Exception as e:
raise e
trace = traceback.format_exc()
if trace.find("already exists") == -1:
nError += 1
print("There was a problem to persist the file " + detPage[key]['filename'] + " in the database!")
if createLog:
logFile.write(
str(nError) + ". There was a problem to persist the file " + detPage[key]['filename'] + " in the database.\n")
persistError = True
con.rollback()
if not persistError:
# move description files of completed folder
source = line2.replace(os.path.basename(line2), "") + detPage[key]['filename']
destination = line2.replace(os.path.basename(line2), "") + r'Read/'
try:
shutil.move(source, destination)
num_persisted_moved += 1
except:
print("There was a problem to move the file " + detPage[key]['filename'] + " in the Description section!")
nError += 1
if createLog:
logFile.write(
str(nError) + ". There was a problem to move the file " + detPage[key]['filename'] + " in the Description section!.\n")
moveError = True
# if the associated description page is not read or not parsed
else:
# query database
# if the product already exists:
# num_in_db += 1
pass
# if number of products on listing page is equal to
# the number of merged, persisted, and moved products plus
# the number of products already in the database
if not persistError and not moveError and len(rw) == (num_persisted_moved + num_in_db):
# move listing file to completed folder
source = line1
destination = line1.replace(os.path.basename(line1), "") + r'Read/'
try:
shutil.move(source, destination)
except:
nError += 1
print("There was a problem to move the file " + line1 + " in the Listing section!")
if createLog:
logFile.write(str(nError) + ". There was a problem to move the file " + line1 + " in the Listing section!.\n")
# g.close ()
if createLog:
logFile.close()
# end = time.time()
# finalTime = float(end-ini)
# print (marketPlace + " Parsing Perfomed Succesfully in %.2f" %finalTime + "!")
input("Parsing the " + marketPlace + " marketplace and data classification done successfully. Press ENTER to continue\n")