dark_web_forums/MarketPlaces/Initialization/prepare_parser.py

__author__ = 'Helium'

import glob
import os
import codecs
import shutil
from MarketPlaces.DB_Connection.db_connection import *
from MarketPlaces.DarkFox.parser import *
from MarketPlaces.Tor2door.parser import *
from MarketPlaces.Apocalypse.parser import *
from MarketPlaces.ThiefWorld.parser import *
from MarketPlaces.AnonymousMarketplace.parser import *
from MarketPlaces.ViceCity.parser import *
from MarketPlaces.TorBay.parser import *
from MarketPlaces.M00nkeyMarket.parser import *
from MarketPlaces.DarkMatter.parser import *
from MarketPlaces.DigitalThriftShop.parser import *
from MarketPlaces.LionMarketplace.parser import *
from MarketPlaces.TorMarket.parser import *
from MarketPlaces.HiddenMarket.parser import *
from MarketPlaces.RobinhoodMarket.parser import *

from MarketPlaces.Classifier.classify_product import predict


def mergePages(rmm, rec):

    # key = u"Pr:" + rec[1].upper() + u" Vendor:" + rec[18].upper()
    # key = rec[23]

    print("----------------- Matched: " + rec[4] + "--------------------")

    if rec[1] == "-1":          # name_vendor
        rec[1] = rmm[0]
    if rec[2] == "-1":          # rating_vendor
        rec[2] = rmm[1]
    if rec[3] == "-1":          # success_vendor
        rec[3] = rmm[2]
    if rec[4] == "-1":          # name_item
        rec[4] = rmm[3]
    if rec[5] == "-1":          # description_item
        rec[5] = rmm[4]
    if rec[6] == "-1":          # cve_item
        rec[6] = rmm[5]
    if rec[7] == "-1":          # ms_item
        rec[7] = rmm[6]
    if rec[8] == "-1":          # category_item
        rec[8] = rmm[7]
    if rec[9] == "-1":          # views_item
        rec[9] = rmm[8]
    if rec[10] == "-1":         # reviews_item
        rec[10] = rmm[9]
    if rec[11] == "-1":         # rating_item
        rec[11] = rmm[10]
    if rec[12] == "-1":         # adddate_item
        rec[12] = rmm[11]
    if rec[13] == "-1":         # btc_item
        rec[13] = rmm[12]
    if rec[14] == "-1":         # usd_item
        rec[14] = rmm[13]
    if rec[15] == "-1":         # euro_item
        rec[15] = rmm[14]
    if rec[16] == "-1":         # quantitysold_item
        rec[16] = rmm[15]
    if rec[17] == "-1":         # quantityleft_item
        rec[17] = rmm[16]
    if rec[18] == "-1":         # shippedfrom_item
        rec[18] = rmm[17]
    if rec[19] == "-1":         # shippedto_item
        rec[19] = rmm[18]

    return rec


def persist_data(url, row, cur):

    marketPlace = create_marketPlace(cur, row, url)

    vendor = create_vendor(cur, row, marketPlace)

    create_items(cur, row, marketPlace, vendor)


def new_parse(marketPlace, url, createLog):

    from MarketPlaces.Initialization.markets_mining import config, CURRENT_DATE

    print("Parsing the " + marketPlace + " marketplace and conduct data classification to store the information in the database.")

    # ini = time.time()

    # Connecting to the database
    con = connectDataBase()
    cur = con.cursor()

    # Creating the tables (The database should be created manually)
    create_database(cur, con)

    nError = 0

    lines = []  # listing pages
    lns = []    # description pages
    detPage = {}

    #Creating the log file for each Market Place
    if createLog:
        if not os.path.exists("./" + marketPlace + "/Logs/" + marketPlace + "_" + CURRENT_DATE + ".log"):
            logFile = open("./" + marketPlace + "/Logs/" + marketPlace + "_" + CURRENT_DATE + ".log", "w")
        else:
            print("Files of the date " + CURRENT_DATE + " from the Market Place " + marketPlace +
                  " were already read. Delete the referent information in the Data Base and also delete the log file"
                  " in the _Logs folder to read files from this Market Place of this date again.")
            raise SystemExit

    mainDir = os.path.join(config.get('Project', 'shared_folder'), "MarketPlaces/" + marketPlace + "/HTML_Pages")

    # Reading the Listing Html Pages
    for fileListing in glob.glob(os.path.join(mainDir, CURRENT_DATE + "\\Listing", '*.html')):
        lines.append(fileListing)

    # Reading the Description Html Pages
    for fileDescription in glob.glob(os.path.join(mainDir, CURRENT_DATE + "\\Description", '*.html')):
        lns.append(fileDescription)

    # Parsing the Description Pages and put the tag's content into a dictionary (Hash table)
    for index, line2 in enumerate(lns):

        print("Reading description folder of '" + marketPlace + "', file '" + os.path.basename(line2) + "', index= " + str(index + 1) + " ... " + str(len(lns)))

        try:
            html = codecs.open(line2.strip('\n'), encoding='utf8')
            soup = BeautifulSoup(html, "html.parser")
            html.close()
        except:

            try:
                html = open(line2.strip('\n'))
                soup = BeautifulSoup(html, "html.parser")
                html.close()
            except:

                nError += 1
                print("There was a problem to read the file " + line2 + " in the Description section!")
                if createLog:
                    logFile.write(str(nError) + ". There was a problem to read the file " + line2 + " in the Description section.\n")
                continue

        try:

            if marketPlace == "DarkFox":
                rmm = darkfox_description_parser(soup)
            elif marketPlace == "Tor2door":
                rmm = tor2door_description_parser(soup)
            elif marketPlace == "Apocalypse":
                rmm = apocalypse_description_parser(soup)
            elif marketPlace == "ThiefWorld":
                rmm = thiefWorld_description_parser(soup)
            elif marketPlace =="AnonymousMarketplace":
                rmm = anonymousMarketplace_description_parser(soup)
            elif marketPlace == "ViceCity":
                rmm = vicecity_description_parser(soup)
            elif marketPlace == "TorBay":
                rmm = torbay_description_parser(soup)
            elif marketPlace == "M00nkeyMarket":
                rmm = m00nkey_description_parser(soup)
            elif marketPlace == "HiddenMarket":
                rmm = hiddenmarket_description_parser(soup)
            elif marketPlace == "DarkMatter":
                rmm = darkmatter_description_parser(soup)
            elif marketPlace == "DigitalThriftShop":
                rmm = digitalThriftShop_description_parser(soup)
            elif marketPlace == "LionMarketplace":
                rmm = lionmarketplace_description_parser(soup)
            elif marketPlace == "TorMarket":
                rmm = tormarket_description_parser(soup)
            elif marketPlace == "RobinhoodMarket":
                rmm = Robinhood_description_parser(soup)
            # key = u"Pr:" + rmm[0].upper()[:desc_lim1] + u" Vendor:" + rmm[13].upper()[:desc_lim2]
            key = u"Url:" + os.path.basename(line2).replace(".html", "")

            # save file address with description record in memory
            detPage[key] = {'rmm': rmm, 'filename': os.path.basename(line2)}

        except Exception as e:
            raise e

            nError += 1
            print("There was a problem to parse the file " + line2 + " in the Description section!")
            if createLog:
                logFile.write(str(nError) + ". There was a problem to parse the file " + line2 + " in the Description section.\n")

    # Parsing the Listing Pages and put the tag's content into a list
    for index, line1 in enumerate(lines):

        print("Reading listing folder of '" + marketPlace + "', file '" + os.path.basename(line1) + "', index= " + str(index + 1) + " ... " + str(len(lines)))

        readError = False
        try:
            html = codecs.open(line1.strip('\n'), encoding='utf8')
            soup = BeautifulSoup(html, "html.parser")
            html.close()
        except:

            try:
                html = open(line1.strip('\n'))
                soup = BeautifulSoup(html, "html.parser")
                html.close()
            except Exception as e:
                raise e
                nError += 1
                print("There was a problem to read the file " + line1 + " in the Listing section!")
                if createLog:
                    logFile.write(str(nError) + ". There was a problem to read the file " + line1 + " in the Listing section.\n")
                readError = True

        if not readError:

            parseError = False
            try:

                if marketPlace == "DarkFox":
                    rw = darkfox_listing_parser(soup)
                elif marketPlace == "Tor2door":
                    rw = tor2door_listing_parser(soup)
                elif marketPlace == "Apocalypse":
                    rw = apocalypse_listing_parser(soup)
                elif marketPlace == "ThiefWorld":
                    rw = thiefWorld_listing_parser(soup)
                elif marketPlace == "AnonymousMarketplace":
                    rw = anonymousMarketplace_listing_parser(soup)
                elif marketPlace == "ViceCity":
                    rw = vicecity_listing_parser(soup)
                elif marketPlace == "TorBay":
                    rw = torbay_listing_parser(soup)
                elif marketPlace == "M00nkeyMarket":
                    rw = m00nkey_listing_parser(soup)
                elif marketPlace == "HiddenMarket":
                    rw =hiddenmarket_listing_parser(soup)
                elif marketPlace == "DarkMatter":
                    rw = darkmatter_listing_parser(soup)
                elif marketPlace == "DigitalThriftShop":
                    rw = digitalThriftShop_listing_parser(soup)
                elif marketPlace == "LionMarketplace":
                    rw = lionmarketplace_listing_parser(soup)
                elif marketPlace == "TorMarket":
                    rw = tormarket_listing_parser(soup)
                elif marketPlace == "RobinhoodMarket":
                    rw = Robinhood_listing_parser(soup)
                else:
                    parseError = True

            except Exception as e:

                nError += 1
                print("There was a problem to parse the file " + line1 + " in the listing section!")
                if createLog:
                    logFile.write(
                        str(nError) + ". There was a problem to parse the file " + line1 + " in the Listing section.\n")
                parseError = True

            if not parseError:

                persistError = False
                moveError = False
                num_in_db = 0
                num_persisted_moved = 0

                for rec in rw:

                    rec = rec.split(',')

                    # if len(detPage) > 0: #It was created here just because Zeroday Market does not have Description Pages
                    #   key = rec[23]

                    # key = u"Pr:" + rec[1].upper()[:list_lim1] + u" Vendor:" + rec[18].upper()[:list_lim2]
                    key = u"Url:" + cleanLink(rec[20])

                    # if the associated description page is parsed
                    if key in detPage:

                        # rec = mergePages(detPage, rec)

                        # Combining the information from Listing and Description Pages
                        rmm = detPage[key]['rmm']
                        rec = mergePages(rmm, rec)

                        # Append to the list the classification of the product
                        # rec.append(str(predict(rec[1], rec[5], language='markets')))
                        rec.append(str(predict(rec[4], rec[5], language='sup_english')))

                        # Persisting the information in the database
                        try:
                            persist_data(url, tuple(rec), cur)
                            con.commit()
                        except Exception as e:

                            trace = traceback.format_exc()

                            if trace.find("already exists") == -1:
                                nError += 1
                                print("There was a problem to persist the file " + detPage[key]['filename'] + " in the database!")
                                if createLog:
                                    logFile.write(
                                        str(nError) + ". There was a problem to persist the file " + detPage[key]['filename'] + " in the database.\n")
                                persistError = True

                            con.rollback()

                        if not persistError:

                            # move description files of completed folder
                            source = line2.replace(os.path.basename(line2), "") + detPage[key]['filename']
                            destination = line2.replace(os.path.basename(line2), "") + r'Read/'

                            try:
                                shutil.move(source, destination)
                                num_persisted_moved += 1
                            except:

                                print("There was a problem to move the file " + detPage[key]['filename'] + " in the Description section!")
                                nError += 1
                                if createLog:
                                    logFile.write(
                                        str(nError) + ". There was a problem to move the file " + detPage[key]['filename'] + " in the Description section!.\n")
                                moveError = True

                    # if the associated description page is not read or not parsed
                    else:
                        # query database
                        # if the product already exists:
                        #     num_in_db += 1
                        pass

                # if number of products on listing page is equal to
                # the number of merged, persisted, and moved products plus
                # the number of products already in the database
                if not persistError and not moveError and len(rw) == (num_persisted_moved + num_in_db):

                    # move listing file to completed folder
                    source = line1
                    destination = line1.replace(os.path.basename(line1), "") + r'Read/'

                    try:
                        shutil.move(source, destination)
                    except:

                        nError += 1
                        print("There was a problem to move the file " + line1 + " in the Listing section!")
                        if createLog:
                            logFile.write(str(nError) + ". There was a problem to move the file " + line1 + " in the Listing section!.\n")

    # g.close ()

    if createLog:
        logFile.close()

    # end = time.time()

    # finalTime = float(end-ini)

    # print (marketPlace + " Parsing Perfomed Succesfully in %.2f" %finalTime + "!")
    input("Parsing the " + marketPlace + " marketplace and data classification done successfully. Press ENTER to continue\n")