__author__ = 'DarkWeb'
import codecs
import glob
import os
import shutil
from Forums.DB_Connection.db_connection import *
from Forums.BestCardingWorld.parser import *
from Forums.CryptBB.parser import *
from Forums.Classifier.classify_product import predict
#from DarkWebMining_Sample.Forums.Classifier.classify_product import predict_semi
# determines if forum is russian, not really used now but maybe later
def isRussianForum(forum):
with open('russian_forums.txt') as f:
forums = f.readlines()
result = False
for iforum in forums:
iforum = iforum.replace('\n','')
if iforum == forum:
result = True
break
return result
#tries to match description pages to listing pages by using a key made for every description page and every link in listing page
#once verified and matched, the info is merged into a 'rec', which is returned
#@param: detPage is a list of keys of valid pages, rec is the row of data of an instance
#return: rec, row of data, that may have additional data added on after matching description to listing page
def mergePages(rmm, rec):
# key = u"Top:" + rec[1].upper().strip() + u" User:" + rec[5].upper().strip()
# key = rec[16]
print ("----------------- Matched: " + rec[1] + "--------------------")
rec[8] = rmm[1]
rec[9] = rmm[2]
rec[10] = rmm[3]
rec[11] = rmm[4]
rec[12] = rmm[5]
rec[13] = rmm[6]
rec[14] = rmm[7]
rec[15] = rmm[8]
return rec
#gets a string of posts and joins them together into one string to be put in the database as one string of text
#@param: list of strings (the posts of a thread)
#return: string containing the concatenation of all the strings
def getPosts(posts):
strPosts = ' '.join(posts)
return strPosts.strip()
#uses db connection , another program, methods to persists values to the correct categories
#@param: row is the list of entries for this instance, cur is the db connection object
def persist_data(row, cur):
user = create_user(cur, row[5])
forum = create_forum(cur, row)
board = create_board(cur, row, forum)
topic = create_topic(cur, row, forum, board, user)
create_posts(cur, row, forum, board, topic)
#main method for this program, what actually gets the parsed info from the parser, and persists them into the db
#calls the different parser methods here depending on the type of html page
def new_parse(forum, createLog):
print("Parsing The " + forum + " Forum and conduct data classification to store the information in the database.")
crawlerDate = date.today()
ini = time.time()
global site
# Connecting to the database
con = connectDataBase()
cur = con.cursor()
# Creating the tables (The database should be created manually)
create_database(cur, con)
nError = 0
lines = [] #lines.clear()
lns = [] #lns.clear()
detPage = {}
rw = []
# Creating the log file for each Forum
if createLog:
if not os.path.exists("./" + forum + "/Logs/" + forum + "_" + str("%02d" %crawlerDate.month) + str("%02d" %crawlerDate.day) + str("%04d" %crawlerDate.year) + ".log"):
logFile = open("./" + forum + "/Logs/" + forum + "_" + str("%02d" %crawlerDate.today().month) + str("%02d" %crawlerDate.day) + str("%04d" %crawlerDate.year) + ".log", "w")
else:
print("Files of the date " + str("%02d" %crawlerDate.today().month) + str("%02d" %crawlerDate.today().day) + str("%04d" %crawlerDate.today().year) +
" from the Forum " + forum + " were already read. Delete the referent information in the Data Base and also delete the log file "
"in the _Logs folder to read files from this Forum of this date again.")
raise SystemExit
# Reading the Listing Html Pages
for fileListing in glob.glob(os.path.join (os.getcwd().replace("Initialization","") + forum + "\\HTML_Pages\\" + str("%02d" %crawlerDate.month) + str("%02d" %crawlerDate.day) + str("%04d" %crawlerDate.year) + "\\Listing" ,'*.html')):
lines.append(fileListing)
# Reading the Description Html Pages
for fileDescription in glob.glob(os.path.join (os.getcwd().replace("Initialization","") + forum + "\\HTML_Pages\\" + str("%02d" %crawlerDate.month) + str("%02d" %crawlerDate.day) + str("%04d" %crawlerDate.year) + "\\Description" ,'*.html')):
lns.append(fileDescription)
# Parsing the Description Pages and put the tag's content into a dictionary (Hash table)
for index, line2 in enumerate(lns):
print("Reading description folder of '" + forum + "', file '" + os.path.basename(line2) + "', index= " + str(index + 1) + " ... " + str(len(lns)))
try:
html = codecs.open(line2.strip('\n'), encoding='utf8')
soup = BeautifulSoup(html, "html.parser")
html.close()
except:
try:
html = open(line2.strip('\n'))
soup = BeautifulSoup(html, "html.parser")
html.close()
except:
nError += 1
print("There was a problem to read the file " + line2 + " in the Description section!")
if createLog:
logFile.write(str(nError) + ". There was a problem to read the file " + line2 + " in the Description section!\n")
continue
try:
if forum == "BestCardingWorld":
rmm = bestcardingworld_description_parser(soup)
elif forum == "CryptBB":
rmm = cryptBB_description_parser(soup)
# key = u"Top:" + rmm[0].upper().strip() + u" User:" + rmm[2][0].upper().strip()
key = u"Url:" + os.path.basename(line2).replace(".html", "")
# save file address with description record in memory
detPage[key] = {'rmm': rmm, 'filename': os.path.basename(line2)}
except:
nError += 1
print("There was a problem to parse the file " + line2 + " in the Description section!")
if createLog:
logFile.write(str(nError) + ". There was a problem to parse the file " + line2 + " in the Description section.\n")
# Parsing the Listing Pages and put the tag's content into a list
for index, line1 in enumerate(lines):
print("Reading listing folder of '" + forum + "', file '" + os.path.basename(line1) + "', index= " + str(index + 1) + " ... " + str(len(lines)))
readError = False
try:
html = codecs.open(line1.strip('\n'), encoding='utf8')
soup = BeautifulSoup(html, "html.parser")
html.close()
except:
try:
html = open(line1.strip('\n'))
soup = BeautifulSoup(html, "html.parser")
html.close()
except:
nError += 1
print("There was a problem to read the file " + line1 + " in the Listing section!")
if createLog:
logFile.write(str(nError) + ". There was a problem to read the file " + line1 + " in the Listing section.\n")
readError = True
if not readError:
parseError = False
try:
if forum == "BestCardingWorld":
rw = bestcardingworld_listing_parser(soup)
elif forum == "CryptBB":
rw = cryptBB_listing_parser(soup)
except:
nError += 1
print("There was a problem to read the file " + line1 + " in the listing section!")
traceback.print_exc()
if createLog:
logFile.write(
str(nError) + ". There was a problem to read the file " + line1 + " in the Listing section.\n")
parseError = True
if not parseError:
persistError = False
moveError = False
num_in_db = 0
num_persisted_moved = 0
for rec in rw:
rec = rec.split(',')
# key = u"Top:" + rec[1].upper().strip() + u" User:" + rec[5].upper().strip()
# key = rec[16]
url = ''.join(e for e in rec[16] if e.isalnum())
key = u"Url:" + url
if key in detPage:
# Combining the information from Listing and Description Pages
rmm = detPage[key]['rmm']
rec = mergePages(rmm, rec)
# Append to the list the classification of the topic
# if isRussianForum(forum):
# rec.append(str(predict(rec[1], getPosts(rec[8]), language='sup_russian')))
# else:
# rec.append(str(predict(rec[1], getPosts(rec[8]), language='sup_english')))
rec.append(str(predict(rec[1], getPosts(rec[8]), language='sup_english')))
# Persisting the information in the database
try:
persist_data(tuple(rec), cur)
con.commit()
except:
trace = traceback.format_exc()
if trace.find("already exists") == -1:
nError += 1
print("There was a problem to persist the file " + detPage[key]['filename'] + " in the database!")
if createLog:
logFile.write(
str(nError) + ". There was a problem to persist the file " + detPage[key]['filename'] + " in the database.\n")
persistError = True
con.rollback()
if not persistError:
# move description files of completed folder
source = line2.replace(os.path.basename(line2), "") + detPage[key]['filename']
destination = line2.replace(os.path.basename(line2), "") + r'Read/'
try:
shutil.move(source, destination)
num_persisted_moved += 1
except:
print("There was a problem to move the file " + detPage[key]['filename'] + " in the Description section!")
nError += 1
if createLog:
logFile.write(
str(nError) + ". There was a problem to move the file " + detPage[key]['filename'] + " in the Description section!.\n")
moveError = True
# if the associated description page is not read or not parsed
else:
# query database
# if the post already exists:
# num_in_db += 1
pass
# if number of topics on listing page is equal to
# the number of merged, persisted, and moved topics plus
# the number of topics already in the database
if not persistError and not moveError and len(rw) == (num_persisted_moved + num_in_db):
# move listing file to completed folder
source = line1
destination = line1.replace(os.path.basename(line1), "") + r'Read/'
try:
shutil.move(source, destination)
except:
nError += 1
print("There was a problem to move the file " + line1 + " in the Listing section!")
if createLog:
logFile.write(str(nError) + ". There was a problem to move the file " + line1 + " in the Listing section!.\n")
if createLog:
logFile.close()
#end = time.time()
#finalTime = float(end-ini)
#print (forum + " Parsing Perfomed Succesfully in %.2f" %finalTime + "!")
input("Parsing the " + forum + " forum and data classification done successfully. Press ENTER to continue\n")