__author__ = 'DarkWeb'

import psycopg2
import traceback
from MarketPlaces.Utilities.utilities import *
from dateutil.relativedelta import relativedelta, FR
from scipy.spatial import distance

def connectDataBase():

    from MarketPlaces.Initialization.markets_mining import config

    try:

        ip = config.get('PostgreSQL', 'ip')
        username = config.get('PostgreSQL', 'username')
        password = config.get('PostgreSQL', 'password')
        database = config.get('PostgreSQL', 'database')

        return psycopg2.connect(host=ip, user=username, password=password, dbname=database)

    except:

        print("Data base " + config.get('PostgreSQL', 'database') + " not found.")
        raise SystemExit


def verifyMarketPlace(cur, nameMarket):

    try:

        cur.execute("lock table marketPlaces IN ACCESS EXCLUSIVE MODE")

        cur.execute("select market_id from marketPlaces where name_market = %(nameMarket)s limit 1", {'nameMarket': nameMarket})

        recset = cur.fetchall()

        if recset:
            return recset[0]['market_id']
        else:
            return 0

    except:

        trace = traceback.format_exc()
        print (trace)


def verifyVendor(cur, nameVendor, marketId):

    try:

        cur.execute("lock table vendors IN ACCESS EXCLUSIVE MODE")

        cur.execute("select vendor_id from vendors where name_vendor = %(nameVendor)s and market_id = %(marketId)s "
                    "limit 1", {'nameVendor': nameVendor, 'marketId': marketId})

        recset = cur.fetchall()

        if recset:
            return recset[0]['vendor_id']
        else:
            return 0

    except:

        trace = traceback.format_exc()
        print (trace)

def verifyItem(cur, marketId, hrefItem):

    try:

        cur.execute("lock table items IN ACCESS EXCLUSIVE MODE")

        cur.execute("select item_id from items where market_id = %(marketId)s and href_item = %(hrefItem)s limit 1",
                    {'marketId': marketId, 'hrefItem': hrefItem})

        recset = cur.fetchall()

        if recset:
            return recset[0]['item_id']
        else:
            return 0

    except:

        trace = traceback.format_exc()
        print (trace)

def verifyImage(cur, base64Image):

    try:

        cur.execute("lock table market_images IN ACCESS EXCLUSIVE MODE")

        cur.execute("select image_id from market_images where hash_image = %(hashImage)s limit 1",
                    {'hashImage': generate_image_hash(base64Image)})

        recset = cur.fetchall()

        if recset:
            return recset[0]['image_id']
        else:
            return 0

    except:

        trace = traceback.format_exc()
        print (trace)

def getLastMarketPlace(cur):

    try:

        cur.execute("select market_id from marketPlaces order by market_id desc limit 1")

        recset = cur.fetchall()

        if recset:
            return recset[0]['market_id']
        else:
            return 0

    except:

        trace = traceback.format_exc()
        print (trace)


def getLastVendor(cur):

    try:

        cur.execute("select vendor_id from vendors order by vendor_id desc limit 1")

        recset = cur.fetchall()

        if recset:
            return recset[0]['vendor_id']
        else:
            return 0

    except:

        trace = traceback.format_exc()
        print (trace)

def getLastVendorVersion(cur, vendorId):

    try:

        cur.execute("select version_vendor from vendors_history where vendor_id = %(vendorId)s order by version_vendor desc limit 1", {'vendorId': vendorId})

        recset = cur.fetchall()

        if recset:
            return recset[0]['version_vendor']
        else:
            return 0

    except:

        trace = traceback.format_exc()
        print (trace)

def getLastItem(cur):

    try:

        cur.execute("select item_id from items order by item_id desc limit 1")

        recset = cur.fetchall()

        if recset:
            return recset[0]['item_id']
        else:
            return 0

    except:

        trace = traceback.format_exc()
        print (trace)


def getLastItemVersion(cur, itemId):
    try:

        cur.execute("select version_item from items_history where item_id = %(itemId)s order by version_item desc limit 1",
                    {'itemId': itemId})

        recset = cur.fetchall()

        if recset:
            return recset[0]['version_item']
        else:
            return 0

    except:

        trace = traceback.format_exc()
        print(trace)


def getLastImage(cur):

    try:

        cur.execute("select image_id from market_images order by image_id desc limit 1")

        recset = cur.fetchall()

        if recset:
            return recset[0]['image_id']
        else:
            return 0

    except:

        trace = traceback.format_exc()
        print (trace)


def create_marketPlace(cur, row, url):

    marketId = verifyMarketPlace(cur, row[0])

    if not marketId:
       marketId = int(getLastMarketPlace(cur) + 1)

       sql = "Insert into marketplaces (market_id, name_market, url_market, dateinserted_market) " \
             "Values (%s, %s, %s, %s)"

       recset = [marketId, row[0], url, row[23]]

       cur.execute(sql, recset)

    return marketId


def create_vendor(cur, row, marketId):

    vendorId = verifyVendor(cur, row[1], marketId)

    if not vendorId:
       vendorId = int(getLastVendor(cur) + 1)
       newVendor = True
    else:
       newVendor = False

    if newVendor:

        imageId = create_image(cur, row[21], row[23])

        sql = "Insert into vendors (vendor_id, market_id, name_vendor, rating_vendor, successfultransactions_vendor, image_vendor, dateinserted_vendor) " \
              "Values (%s, %s, %s, %s, %s, %s, %s)"

        recset = [vendorId, marketId,
                  row[1],
                  row[2] if row[2] != '-1' else None,
                  row[3] if row[3] != '-1' else None,
                  imageId,
                  row[23]]

        cur.execute(sql, recset)

    else:

        # Tracking potential vendor changes
        sql = "select * from vendors where vendor_id = %(vendorId)s"
        cur.execute(sql, {'vendorId': vendorId})

        recset = cur.fetchall()

        imageId = recset[0]['image_vendor']
        if not is_same_image(cur, row[21], recset[0]['image_vendor']):
            imageId = create_image(cur, row[21], row[23])

        if (str(recset[0]['rating_vendor']) != str(row[2] if row[2] != '-1' else None) or # there was a change in the vendor information
            str(recset[0]['successfultransactions_vendor']) != str(row[3] if row[3] != '-1' else None) or
            str(recset[0]['image_vendor']) != str(imageId)):

            vendorVersionId = int(getLastVendorVersion(cur, vendorId) + 1)

            sql = "Insert into vendors_history (vendor_id, version_vendor, market_id, name_vendor, rating_vendor, " \
                  "successfultransactions_vendor, image_vendor, dateinserted_vendor) " \
                  "Values (%s, %s, %s, %s, %s, %s, %s, %s)"

            recset = [vendorId, vendorVersionId, marketId,
                      recset[0]['name_vendor'],
                      recset[0]['rating_vendor'],
                      recset[0]['successfultransactions_vendor'],
                      recset[0]['image_vendor'],
                      recset[0]['dateinserted_vendor']]

            cur.execute(sql, recset)

            sql = "Update vendors set rating_vendor = %(rating_vendor)s, successfultransactions_vendor = %(successfultransactions_vendor)s, " \
                  "image_vendor = %(image_vendor)s, dateinserted_vendor = %(dateinserted_vendor)s where vendor_id = %(vendorId)s"
            cur.execute(sql, {'rating_vendor': row[2] if row[2] != '-1' else None,
                              'successfultransactions_vendor': row[3] if row[3] != '-1' else None,
                              'image_vendor': imageId,
                              'dateinserted_vendor': row[23],
                              'vendorId': vendorId})

    return vendorId


def create_items(cur, row, marketId, vendorId):

    hrefItem = get_relative_url(row[22])

    itemId = verifyItem(cur, marketId, hrefItem)

    if not itemId:
        itemId = int(getLastItem(cur) + 1)
        newItem = True
    else:
        newItem = False

    if newItem:

        imageId = create_image(cur, row[20], row[23])

        sql = "Insert into items (item_id, market_id, vendor_id, name_item, description_item, cve_item, ms_item, category_item, " \
              "views_item, reviews_item, rating_item, dateadded_item, btc_item, usd_item, euro_item, quantitysold_item, " \
              "quantityleft_item, shippedfrom_item, shippedto_item, image_item, href_item, lastseen_item, dateinserted_item, " \
              "classification_item) Values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, " \
              "%s, %s, %s, %s, %s)"

        recset = [itemId, marketId, vendorId,
                  row[4],
                  row[5] if row[5] != '-1' else None,
                  row[6] if row[6] != '-1' else None,
                  row[7] if row[7] != '-1' else None,
                  row[8] if row[8] != '-1' else None,
                  row[9] if row[9] != '-1' else None,
                  row[10] if row[10] != '-1' else None,
                  row[11] if row[11] != '-1' else None,
                  row[12] if row[12] != '-1' else None,
                  row[13] if row[13] != '-1' else None,
                  row[14] if row[14] != '-1' else None,
                  row[15] if row[15] != '-1' else None,
                  row[16] if row[16] != '-1' else None,
                  row[17] if row[17] != '-1' else None,
                  row[18] if row[18] != '-1' else None,
                  row[19] if row[19] != '-1' else None,
                  imageId,
                  hrefItem,
                  row[23],
                  row[23],
                  row[24] if row[24] != '-1' else None]

        cur.execute(sql, recset)

    else:

            # Tracking potential item changes
            sql = "select * from items where item_id = %(itemId)s"
            cur.execute(sql, {'itemId': itemId})

            recset = cur.fetchall()

            imageId = recset[0]['image_item']
            if not is_same_image(cur, row[20], recset[0]['image_item']):
                imageId = create_image(cur, row[20], row[23])

            if (str(recset[0]['vendor_id']) != str(vendorId) or
                str(recset[0]['name_item']) != str(row[4] if row[4] != '-1' else None) or
                str(recset[0]['description_item']) != str(row[5] if row[5] != '-1' else None) or
                str(recset[0]['cve_item']) != str(row[6] if row[6] != '-1' else None) or
                str(recset[0]['ms_item']) != str(row[7] if row[7] != '-1' else None) or
                str(recset[0]['category_item']) != str(row[8] if row[8] != '-1' else None) or
                str(recset[0]['views_item']) != str(row[9] if row[9] != '-1' else None) or
                str(recset[0]['reviews_item']) != str(row[10] if row[10] != '-1' else None) or
                str(recset[0]['rating_item']) != str(row[11] if row[11] != '-1' else None) or
                str(recset[0]['dateadded_item']) != str(row[12] if row[12] != '-1' else None) or
                str(recset[0]['btc_item']) != str(row[13] if row[13] != '-1' else None) or
                str(recset[0]['usd_item']) != str(row[14] if row[14] != '-1' else None) or
                str(recset[0]['euro_item']) != str(row[15] if row[15] != '-1' else None) or
                str(recset[0]['quantitysold_item']) != str(row[16] if row[16] != '-1' else None) or
                str(recset[0]['quantityleft_item']) != str(row[17] if row[17] != '-1' else None) or
                str(recset[0]['shippedfrom_item']) != str(row[18] if row[18] != '-1' else None) or
                str(recset[0]['shippedto_item']) != str(row[19] if row[19] != '-1' else None) or
                str(recset[0]['image_item']) != str(imageId) or
                str(recset[0]['classification_item']) != str(row[24] if row[24] != '-1' else None)):

                itemVersionId = int(getLastItemVersion(cur, itemId) + 1)

                sql = "Insert into items_history (item_id, version_item, market_id, vendor_id, name_item, description_item, cve_item, ms_item, category_item, " \
                      "views_item, reviews_item, rating_item, dateadded_item, btc_item, usd_item, euro_item, quantitysold_item, " \
                      "quantityleft_item, shippedfrom_item, shippedto_item, image_item, href_item, lastseen_item, dateinserted_item, " \
                      "classification_item) Values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, " \
                      "%s, %s, %s, %s, %s, %s)"

                recset = [itemId, itemVersionId, marketId,
                         recset[0]['vendor_id'],
                         recset[0]['name_item'],
                         recset[0]['description_item'],
                         recset[0]['cve_item'],
                         recset[0]['ms_item'],
                         recset[0]['category_item'],
                         recset[0]['views_item'],
                         recset[0]['reviews_item'],
                         recset[0]['rating_item'],
                         recset[0]['dateadded_item'],
                         recset[0]['btc_item'],
                         recset[0]['usd_item'],
                         recset[0]['euro_item'],
                         recset[0]['quantitysold_item'],
                         recset[0]['quantityleft_item'],
                         recset[0]['shippedfrom_item'],
                         recset[0]['shippedto_item'],
                         recset[0]['image_item'],
                         recset[0]['href_item'],
                         recset[0]['lastseen_item'],
                         recset[0]['dateinserted_item'],
                         recset[0]['classification_item']]

                cur.execute(sql, recset)

                sql = "Update items set vendor_id = %(vendor_id)s, name_item = %(name_item)s, " \
                      "description_item = %(description_item)s, cve_item = %(cve_item)s, ms_item = %(ms_item)s, " \
                      "category_item = %(category_item)s, views_item = %(views_item)s, reviews_item = %(reviews_item)s, " \
                      "rating_item = %(rating_item)s, dateadded_item = %(dateadded_item)s, btc_item = %(btc_item)s, " \
                      "usd_item = %(usd_item)s, euro_item = %(euro_item)s, quantitysold_item = %(quantitysold_item)s, " \
                      "quantityleft_item = %(quantityleft_item)s, shippedfrom_item = %(shippedfrom_item)s, shippedto_item = %(shippedto_item)s, " \
                      "image_item = %(image_item)s, lastseen_item = %(lastseen_item)s, dateinserted_item = %(dateinserted_item)s, " \
                      "classification_item = %(classification_item)s where item_id = %(itemId)s"

                cur.execute(sql, {'vendor_id': vendorId,
                                 'name_item': row[4] if row[4] != '-1' else None,
                                 'description_item': row[5] if row[5] != '-1' else None,
                                 'cve_item': row[6] if row[6] != '-1' else None,
                                 'ms_item': row[7] if row[7] != '-1' else None,
                                 'category_item': row[8] if row[8] != '-1' else None,
                                 'views_item': row[9] if row[9] != '-1' else None,
                                 'reviews_item': row[10] if row[10] != '-1' else None,
                                 'rating_item': row[11] if row[11] != '-1' else None,
                                 'dateadded_item': row[12] if row[12] != '-1' else None,
                                 'btc_item': row[13] if row[13] != '-1' else None,
                                 'usd_item': row[14] if row[14] != '-1' else None,
                                 'euro_item': row[15] if row[15] != '-1' else None,
                                 'quantitysold_item': row[16] if row[16] != '-1' else None,
                                 'quantityleft_item': row[17] if row[17] != '-1' else None,
                                 'shippedfrom_item': row[18] if row[18] != '-1' else None,
                                 'shippedto_item': row[19] if row[19] != '-1' else None,
                                 'image_item': imageId,
                                 'lastseen_item': row[23],
                                 'dateinserted_item': row[23],
                                 'classification_item': row[24] if row[24] != '-1' else None,
                                 'itemId': itemId})

            else: #updating when was the last time the crawler saw that item

                sql = "Update items set lastseen_item = %(lastseen_item)s where item_id = %(itemId)s"

                cur.execute(sql, {'lastseen_item': row[23],
                                  'itemId': itemId})

    return itemId

def is_same_image(cur, newBase64Image, oldImageId):

    if newBase64Image == "-1" and str(oldImageId) == str(None):
        return True

    if newBase64Image == "-1": # and str(oldImageId) != str(None)
        return False

    if str(oldImageId) == str(None): # and newBase64Image != "-1"
        return False

    sql = "select * from market_images where image_id = %(imageId)s"
    cur.execute(sql, {'imageId': oldImageId})

    recset = cur.fetchall()

    decImage = decode_decrypt_image_in_base64(newBase64Image)

    hash1 = generate_image_hash(decImage)
    hash2 = recset[0]['hash_image']

    if hash1 == hash2:
        return True
    
    features1 = json.loads(extract_hidden_layer_output(decImage))
    features2 = json.loads(recset[0]['resnet50_image'])

    similarity = 1 - distance.cosine(features1, features2)

    return similarity >= 0.8

def create_image(cur, base64Image, date):

    if base64Image == "-1":
        return None

    decImage = decode_decrypt_image_in_base64(base64Image)

    imageId = verifyImage(cur, decImage)

    if not imageId:
       imageId = int(getLastImage(cur) + 1)
       newImage = True
    else:
       newImage = False

    if newImage:

        hashImage = generate_image_hash(decImage)
        hloFeatures = extract_hidden_layer_output(decImage)
        keypoints, descriptors = extract_keypoints(decImage)

        sql = "Insert into market_images(image_id, hash_image, base64_image, " \
              "resnet50_image, siftkey_image, siftdesc_image, dateinserted_image) " \
              "Values (%s, %s, %s, %s, %s, %s, %s)"

        recset = [imageId,
                  hashImage,
                  base64Image,
                  hloFeatures,
                  keypoints,
                  descriptors,
                  date]

        cur.execute(sql, recset)
    
    return imageId

def create_status(cur, marketId, date, listings, descriptions, status):

    date = datetime.strptime(date, "%m%d%Y")

    # getting last Fridays a reference date
    date_reference = date + relativedelta(weekday=FR(-1))

    # checking if status already exists
    sql = "select * from marketplaces_status where market_id = %(market_id)s and date_inserted = %(date_inserted)s"
    cur.execute(sql, {'market_id': marketId, 'date_inserted': date})

    recset = cur.fetchall()
    if recset:
       sql = "Update marketplaces_status set listings = %(listings)s, descriptions = %(descriptions)s, status = %(status)s, date_reference = %(date_reference)s " \
             "where market_id = %(market_id)s and date_inserted = %(date_inserted)s"
       recset = {'listings': listings, 'descriptions': descriptions, 'status': status, 'date_reference': date_reference, 'market_id': marketId, 'date_inserted': date}
    else:
       sql = "Insert into marketplaces_status (market_id, date_inserted, listings, descriptions, status, date_reference) Values (%s, %s, %s, %s, %s, %s)"
       recset = [marketId, date, listings, descriptions, status, date_reference]

    cur.execute(sql, recset)

def create_database(cur, con):

    try:

        sql = "create table market_images(image_id integer not null, hash_image character varying(64) not null, base64_image text not null, " \
              "resnet50_image text null, siftkey_image text null, siftdesc_image text null, " \
              "dateinserted_image timestamp(6) with time zone not null, " \
              "constraint market_images_pk primary key (image_id))"
        cur.execute(sql)

        sql = "create unique index unique_market_image ON market_images USING btree (hash_image ASC NULLS LAST)"
        cur.execute(sql)

        sql = "create table marketplaces(market_id integer not null, name_market character varying(255) not null, " \
              "url_market character varying(255) not null, dateinserted_market timestamp(6) with time zone not null, " \
              "constraint markets_pk primary key (market_id))"
        cur.execute(sql)

        sql = "create unique index unique_market ON marketplaces USING btree (name_market ASC NULLS LAST)"
        cur.execute(sql)

        sql = "Create table marketplaces_status (market_id integer NOT NULL, date_inserted date NOT NULL, " \
              "listings integer NOT NULL, descriptions integer NOT NULL, status bit(1) NOT NULL, date_reference date NOT NULL, " \
              "CONSTRAINT marketplaces_status_pk PRIMARY KEY (market_id, date_inserted), " \
              "CONSTRAINT marketplaces_status_fk FOREIGN KEY (market_id) REFERENCES marketplaces (market_id))"
        cur.execute(sql)

        sql = "create table vendors(vendor_id integer not null, market_id integer not null, name_vendor character " \
              "varying(255) not null, rating_vendor character varying(255), successfultransactions_vendor integer " \
              "null, image_vendor integer null, dateinserted_vendor timestamp(6) with time zone not null, " \
              "constraint vendors_pk primary key (vendor_id), " \
              "constraint vendors_market_id_fk foreign key (market_id) references marketplaces (market_id), " \
              "constraint vendors_image_id_fkey foreign key (image_vendor) references market_images (image_id))"
        cur.execute(sql)

        sql = "create unique index unique_vendor ON vendors USING btree (market_id ASC NULLS LAST, name_vendor ASC NULLS LAST)"
        cur.execute(sql)

        sql = "create table vendors_history(vendor_id integer not null, version_vendor integer not null, market_id integer not null, name_vendor " \
              "character varying(255) not null, rating_vendor character varying(255), successfultransactions_vendor " \
              "integer null, image_vendor integer null, dateinserted_vendor timestamp(6) with time zone not null, " \
              "constraint vendors_history_pk primary key (vendor_id, version_vendor), " \
              "constraint vendors_history_vendor_id_fkey foreign key (vendor_id) references vendors (vendor_id), " \
              "constraint vendors_history_market_id_fkey foreign key (market_id) references marketplaces (market_id), " \
              "constraint vendors_history_image_id_fkey foreign key (image_vendor) references market_images (image_id))"
        cur.execute(sql)

        sql = "create table items(item_id integer not null, market_id integer not null, vendor_id integer not null, name_item character " \
              "varying(255) not null, description_item character varying(1000000) null, cve_item character varying(" \
              "255) null, ms_item character varying(255) null, category_item character varying(255) null, views_item " \
              "integer null, reviews_item integer null, rating_item character varying(255) null, dateadded_item " \
              "character varying(25) null, btc_item character varying(255) null, usd_item character varying(255) " \
              "null, euro_item character varying(255) null, quantitysold_item integer null, quantityleft_item " \
              "character varying(255) null, shippedfrom_item character varying(255) null, shippedto_item character " \
              "varying(255) null, image_item integer null, href_item character varying(255) not null, " \
              "lastseen_item timestamp(6) with time zone not null, dateinserted_item timestamp(6) with time zone not null, " \
              "classification_item double precision null, " \
              "constraint items_pk primary key (item_id), " \
              "constraint items_market_id_fkey foreign key (market_id) references marketplaces (market_id), " \
              "constraint items_vendor_id_fkey foreign key (vendor_id) references vendors (vendor_id), " \
              "constraint items_image_id_fkey foreign key (image_item) references market_images (image_id))"
        cur.execute(sql)

        sql = "create unique index unique_item ON items USING btree (market_id ASC NULLS LAST, href_item ASC NULLS LAST)"
        cur.execute(sql)

        sql = "create table items_history(item_id integer not null, version_item integer not null, market_id integer not null, " \
              "vendor_id integer not null, name_item character varying(255) not null, description_item character varying(1000000) null, " \
              "cve_item character varying(255) null, ms_item character varying(255) null, category_item character varying(255) null, views_item " \
              "integer null, reviews_item integer null, rating_item character varying(255) null, dateadded_item " \
              "character varying(25) null, btc_item character varying(255) null, usd_item character varying(255) " \
              "null, euro_item character varying(255) null, quantitysold_item integer null, quantityleft_item " \
              "character varying(255) null, shippedfrom_item character varying(255) null, shippedto_item character " \
              "varying(255) null, image_item integer null, href_item character varying(255) not null, " \
              "lastseen_item timestamp(6) with time zone not null, dateinserted_item timestamp(6) with time zone not null, " \
              "classification_item double precision null, " \
              "constraint items_history_pk primary key (item_id, version_item), " \
              "constraint items_history_item_id_fkey foreign key (item_id) references items (item_id), " \
              "constraint items_history_market_id_fkey foreign key (market_id) references marketplaces (market_id), " \
              "constraint items_history_vendor_id_fkey foreign key (vendor_id) references vendors (vendor_id), " \
              "constraint items_history_image_id_fkey foreign key (image_item) references market_images (image_id))"
        cur.execute(sql)

        con.commit()

    except:

        con.rollback()

        trace = traceback.format_exc()

        if (trace.find("already exists")==-1):
            print ("There was a problem during the database creation." )
            traceback.print_exc()
            raise SystemExit
