khangtran
/
dark_web_forums
0
0
Fork 0
Code Issues 0 Pull Requests 0 Projects 0 Releases 0 Wiki Activity
Browse Source

Db connection Markets and Forums.

main
ericssonmarin-cpp 1 year ago
parent
f375e0860f
commit
47c851ebeb
2 changed files with 205 additions and 45 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +3
    -2
      Forums/DB_Connection/db_connection.py
  2. +202
    -43
      MarketPlaces/DB_Connection/db_connection.py

+ 3
- 2
Forums/DB_Connection/db_connection.py View File

@ -271,7 +271,8 @@ def create_topic(cur, forumId, row, authorId):
recset = cur.fetchall()
if (str(recset[0][4]) != str(row[1]) or str(recset[0][5]) != str(row[4]) or str(recset[0][6]) != str(row[5])): # there was a change in the topic information
if (str(recset[0][4]) != str(row[1]) or str(recset[0][5]) != str(row[4] if row[4] != '-1' else None) or # there was a change in the topic information
str(recset[0][6]) != str(row[5] if row[5] != '-1' else None)):
sql = "Insert into topics_history (topic_id, forum_id, author_id, title_topic, board_topic, views_topic, posts_topic, " \
"href_topic, dateadded_topic, dateinserted_topic, classification_topic) Values (%s, %s, %s, %s, %s, " \
@ -352,7 +353,7 @@ def create_user(cur, row, forumId, index):
recset = cur.fetchall()
if (str(recset[0][3]) != str(row[10][index]) or str(recset[0][4]) != str(row[11][index]) or
if (str(recset[0][3]) != str(row[10][index] if row[10][index] != '-1' else None) or str(recset[0][4]) != str(row[11][index] if row[11][index] != '-1' else None) or
str(recset[0][5]) != str(row[12][index] if row[12][index] != '-1' else None) or str(recset[0][6]) != str(row[13][index] if row[13][index] != '-1' else None)): # there was a change in the user information
if (str(recset[0][3]) != 'Nan' or str(recset[0][4]) != 'Nan' or str(recset[0][5]) != 'Nan' or str(recset[0][6]) != 'Nan'):


+ 202
- 43
MarketPlaces/DB_Connection/db_connection.py View File

@ -7,9 +7,9 @@ import configparser
def connectDataBase():
try:
from MarketPlaces.Initialization.markets_mining import config
from MarketPlaces.Initialization.markets_mining import config
try:
ip = config.get('PostgreSQL', 'ip')
username = config.get('PostgreSQL', 'username')
@ -20,7 +20,7 @@ def connectDataBase():
except:
print ("Data base (darkweb_marketplaces) not found.")
print("Data base " + config.get('PostgreSQL', 'database') + " not found.")
raise SystemExit
@ -62,6 +62,24 @@ def verifyVendor(cur, nameVendor, marketId):
trace = traceback.format_exc()
print (trace)
def verifyItem(cur, marketId, vendorId, nameItem):
try:
cur.execute("select item_id from items where market_id = %(marketId)s and vendor_id = %(vendorId)s and name_item = %(nameItem)s"
"limit 1", {'marketId': marketId, 'vendorId': vendorId, 'nameItem': nameItem})
recset = cur.fetchall()
if recset:
return recset[0][0]
else:
return 0
except:
trace = traceback.format_exc()
print (trace)
def getLastMarketPlace(cur):
@ -100,6 +118,23 @@ def getLastVendor(cur):
trace = traceback.format_exc()
print (trace)
def getLastItem(cur):
try:
cur.execute("select item_id from items order by item_id desc limit 1")
recset = cur.fetchall()
if recset:
return recset[0][0]
else:
return 0
except:
trace = traceback.format_exc()
print (trace)
def create_marketPlace(cur, row, url):
@ -124,6 +159,11 @@ def create_vendor(cur, row, marketId):
if not vendorId:
vendorId = int(getLastVendor(cur) + 1)
newVendor = True
else:
newVendor = False
if newVendor:
sql = "Insert into vendors (vendor_id, market_id, name_vendor, rating_vendor, successfultransactions_vendor, dateinserted_vendor) Values (%s, %s, %s, %s, %s, %s)"
@ -135,51 +175,164 @@ def create_vendor(cur, row, marketId):
cur.execute(sql, recset)
else:
# Tracking potential vendor changes
sql = "select * from vendors where vendor_id = %(vendorId)s"
cur.execute(sql, {'vendorId': vendorId})
recset = cur.fetchall()
if (str(recset[0][3]) != str(row[2] if row[2] != '-1' else None) or # there was a change in the vendor information
str(recset[0][4]) != str(row[3] if row[3] != '-1' else None)):
sql = "Insert into vendors_history (vendor_id, market_id, name_vendor, rating_vendor, successfultransactions_vendor, dateinserted_vendor) Values (%s, %s, %s, %s, %s, %s)"
recset = [vendorId, marketId,
recset[0][2],
recset[0][3],
recset[0][4],
recset[0][5]]
cur.execute(sql, recset)
sql = "Update vendors set rating_vendor = %(rating_vendor)s, successfultransactions_vendor = %(successfultransactions_vendor)s, " \
"dateinserted_vendor = %(dateinserted_vendor)s where vendor_id = %(vendorId)s"
cur.execute(sql, {'rating_vendor': row[2] if row[2] != '-1' else None,
'successfultransactions_vendor': row[3] if row[3] != '-1' else None,
'dateinserted_vendor': row[21],
'vendorId': vendorId})
return vendorId
def create_items(cur, row, marketId, vendorId):
sql = "Insert into items (market_id, vendor_id, name_item, description_item, cve_item, ms_item, category_item, " \
"views_item, reviews_item, rating_item, dateadded_item, btc_item, usd_item, euro_item, quantitysold_item, " \
"quantityleft_item, shippedfrom_item, shippedto_item, href_item, lastseen_item, dateinserted_item, " \
"classification_item) Values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, " \
"%s, %s, %s)"
recset = [marketId, vendorId,
row[4],
row[5] if row[5] != '-1' else None,
row[6] if row[6] != '-1' else None,
row[7] if row[7] != '-1' else None,
row[8] if row[8] != '-1' else None,
row[9] if row[9] != '-1' else None,
row[10] if row[10] != '-1' else None,
row[11] if row[11] != '-1' else None,
row[12] if row[12] != '-1' else None,
row[13] if row[13] != '-1' else None,
row[14] if row[14] != '-1' else None,
row[15] if row[15] != '-1' else None,
row[16] if row[16] != '-1' else None,
row[17] if row[17] != '-1' else None,
row[18] if row[18] != '-1' else None,
row[19] if row[19] != '-1' else None,
row[20] if row[20] != '-1' else None,
row[21],
row[21],
row[22]]
cur.execute(sql, recset)
itemId = verifyItem(cur, marketId, vendorId, row[4])
if not itemId:
itemId = int(getLastItem(cur) + 1)
newItem = True
else:
newItem = False
if newItem:
sql = "Insert into items (item_id, market_id, vendor_id, name_item, description_item, cve_item, ms_item, category_item, " \
"views_item, reviews_item, rating_item, dateadded_item, btc_item, usd_item, euro_item, quantitysold_item, " \
"quantityleft_item, shippedfrom_item, shippedto_item, href_item, lastseen_item, dateinserted_item, " \
"classification_item) Values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, " \
"%s, %s, %s, %s)"
recset = [itemId, marketId, vendorId,
row[4],
row[5] if row[5] != '-1' else None,
row[6] if row[6] != '-1' else None,
row[7] if row[7] != '-1' else None,
row[8] if row[8] != '-1' else None,
row[9] if row[9] != '-1' else None,
row[10] if row[10] != '-1' else None,
row[11] if row[11] != '-1' else None,
row[12] if row[12] != '-1' else None,
row[13] if row[13] != '-1' else None,
row[14] if row[14] != '-1' else None,
row[15] if row[15] != '-1' else None,
row[16] if row[16] != '-1' else None,
row[17] if row[17] != '-1' else None,
row[18] if row[18] != '-1' else None,
row[19] if row[19] != '-1' else None,
row[20] if row[20] != '-1' else None,
row[21],
row[21],
row[22]]
cur.execute(sql, recset)
else:
# Tracking potential item changes
sql = "select * from items where item_id = %(itemId)s"
cur.execute(sql, {'itemId': itemId})
recset = cur.fetchall()
if (str(recset[0][4]) != str(row[5] if row[5] != '-1' else None) or str(recset[0][5]) != str(row[6] if row[6] != '-1' else None) or
str(recset[0][6]) != str(row[7] if row[7] != '-1' else None) or str(recset[0][7]) != str(row[8] if row[8] != '-1' else None) or
str(recset[0][8]) != str(row[9] if row[9] != '-1' else None) or str(recset[0][9]) != str(row[10] if row[10] != '-1' else None) or
str(recset[0][10]) != str(row[11] if row[11] != '-1' else None) or str(recset[0][11]) != str(row[12] if row[12] != '-1' else None) or
str(recset[0][12]) != str(row[13] if row[13] != '-1' else None) or str(recset[0][13]) != str(row[14] if row[14] != '-1' else None) or
str(recset[0][14]) != str(row[15] if row[15] != '-1' else None) or str(recset[0][15]) != str(row[16] if row[16] != '-1' else None) or
str(recset[0][16]) != str(row[17] if row[17] != '-1' else None) or str(recset[0][17]) != str(row[18] if row[18] != '-1' else None)):
sql = "Insert into items_history (item_id, market_id, vendor_id, name_item, description_item, cve_item, ms_item, category_item, " \
"views_item, reviews_item, rating_item, dateadded_item, btc_item, usd_item, euro_item, quantitysold_item, " \
"quantityleft_item, shippedfrom_item, shippedto_item, href_item, lastseen_item, dateinserted_item, " \
"classification_item) Values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, " \
"%s, %s, %s, %s)"
recset = [itemId, marketId, vendorId,
recset[0][3],
recset[0][4],
recset[0][5],
recset[0][6],
recset[0][7],
recset[0][8],
recset[0][9],
recset[0][10],
recset[0][11],
recset[0][12],
recset[0][13],
recset[0][14],
recset[0][15],
recset[0][16],
recset[0][17],
recset[0][18],
recset[0][19],
recset[0][20],
recset[0][21],
recset[0][22]]
cur.execute(sql, recset)
sql = "Update items set description_item = %(description_item)s, cve_item = %(cve_item)s, ms_item = %(ms_item)s, " \
"category_item = %(category_item)s, views_item = %(views_item)s, reviews_item = %(reviews_item)s, " \
"rating_item = %(rating_item)s, dateadded_item = %(dateadded_item)s, btc_item = %(btc_item)s, " \
"usd_item = %(usd_item)s, euro_item = %(euro_item)s, quantitysold_item = %(quantitysold_item)s, " \
"quantityleft_item = %(quantityleft_item)s, shippedfrom_item = %(shippedfrom_item)s, shippedto_item = %(shippedto_item)s, " \
"dateinserted_item = %(dateinserted_item)s where item_id = %(itemId)s"
cur.execute(sql, {'description_item': row[5] if row[5] != '-1' else None,
'cve_item': row[6] if row[6] != '-1' else None,
'ms_item': row[7] if row[7] != '-1' else None,
'category_item': row[8] if row[8] != '-1' else None,
'views_item': row[9] if row[9] != '-1' else None,
'reviews_item': row[10] if row[10] != '-1' else None,
'rating_item': row[11] if row[11] != '-1' else None,
'dateadded_item': row[12] if row[12] != '-1' else None,
'btc_item': row[13] if row[13] != '-1' else None,
'usd_item': row[14] if row[14] != '-1' else None,
'euro_item': row[15] if row[15] != '-1' else None,
'quantitysold_item': row[16] if row[16] != '-1' else None,
'quantityleft_item': row[17] if row[17] != '-1' else None,
'shippedfrom_item': row[18] if row[18] != '-1' else None,
'shippedto_item': row[19] if row[19] != '-1' else None,
'dateinserted_item': row[21],
'itemId': itemId})
return itemId
def create_database(cur, con):
try:
sql = "create table marketplaces(market_id integer not null, name_market character varying(255) not null, " \
"url_market character varying(255) null, dateinserted_market timestamp(6) with time zone not null, " \
"url_market character varying(255) not null, dateinserted_market timestamp(6) with time zone not null, " \
"constraint markets_pk primary key (market_id))"
cur.execute(sql)
sql = "create unique index unique_market ON marketplaces USING btree (name_market ASC NULLS LAST)"
cur.execute(sql)
sql = "create table vendors(vendor_id integer not null, market_id integer not null, name_vendor character " \
"varying(255) not null, rating_vendor character varying(255), successfultransactions_vendor integer " \
"null, dateinserted_vendor timestamp(6) with time zone not null, constraint vendors_pk primary key (" \
@ -187,6 +340,9 @@ def create_database(cur, con):
"market_id))"
cur.execute(sql)
sql = "create unique index unique_vendor ON vendors USING btree (market_id ASC NULLS LAST, name_vendor ASC NULLS LAST)"
cur.execute(sql)
sql = "create table vendors_history(vendor_id integer not null, market_id integer not null, name_vendor " \
"character varying(255) not null, rating_vendor character varying(255), successfultransactions_vendor " \
"integer null, dateinserted_vendor timestamp(6) with time zone not null, constraint vendors_history_pk " \
@ -195,33 +351,36 @@ def create_database(cur, con):
"market_id) references marketplaces (market_id))"
cur.execute(sql)
sql = "create table items(market_id integer not null, vendor_id integer not null, name_item character " \
sql = "create table items(item_id integer not null, market_id integer not null, vendor_id integer not null, name_item character " \
"varying(255) not null, description_item character varying(1000000) null, cve_item character varying(" \
"255) null, ms_item character varying(255) null, category_item character varying(255) null, views_item " \
"integer null, reviews_item integer null, rating_item character varying(255) null, dateadded_item " \
"character varying(25) null, btc_item character varying(255) null, usd_item character varying(255) " \
"null, euro_item character varying(255) null, quantitysold_item integer null, quantityleft_item " \
"character varying(255) null, shippedfrom_item character varying(255) null, shippedto_item character " \
"varying(255) null, href_item character varying(255) null, lastseen_item timestamp(6) with time zone " \
"varying(255) null, href_item character varying(255) not null, lastseen_item timestamp(6) with time zone " \
"not null, dateinserted_item timestamp(6) with time zone not null, classification_item double " \
"precision not null, constraint items_pk primary key (market_id, vendor_id, name_item), constraint " \
"precision not null, constraint items_pk primary key (item_id), constraint " \
"items_market_id_fkey foreign key (market_id) references marketplaces (market_id),constraint " \
"items_vendor_id_fkey foreign key (vendor_id) references vendors (vendor_id))"
cur.execute(sql)
sql = "create table items_history(market_id integer not null, vendor_id integer not null, name_item character " \
sql = "create unique index unique_item ON items USING btree (market_id ASC NULLS LAST, vendor_id ASC NULLS LAST, name_item ASC NULLS LAST)"
cur.execute(sql)
sql = "create table items_history(item_id integer not null, market_id integer not null, vendor_id integer not null, name_item character " \
"varying(255) not null, description_item character varying(1000000) null, cve_item character varying(" \
"255) null, ms_item character varying(255) null, category_item character varying(255) null, views_item " \
"integer null, reviews_item integer null, rating_item character varying(255) null, dateadded_item " \
"character varying(25) null, btc_item character varying(255) null, usd_item character varying(255) " \
"null, euro_item character varying(255) null, quantitysold_item integer null, quantityleft_item " \
"character varying(255) null, shippedfrom_item character varying(255) null, shippedto_item character " \
"varying(255) null, href_item character varying(255) null, lastseen_item timestamp(6) with time zone " \
"varying(255) null, href_item character varying(255) not null, lastseen_item timestamp(6) with time zone " \
"not null, dateinserted_item timestamp(6) with time zone not null, classification_item double " \
"precision not null, constraint items_history_pk primary key (market_id, vendor_id, name_item, " \
"dateinserted_item), constraint items_history_market_id_fkey foreign key (market_id) references " \
"marketplaces (market_id), constraint items_history_vendor_id_fkey foreign key (vendor_id) references " \
"vendors (vendor_id))"
"precision not null, constraint items_history_pk primary key (item_id, dateinserted_item), " \
"constraint items_history_market_id_fkey foreign key (market_id) references marketplaces (market_id), " \
"constraint items_history_vendor_id_fkey foreign key (vendor_id) references vendors (vendor_id), " \
"constraint items_history_item_id_fkey foreign key (item_id) references items (item_id))"
cur.execute(sql)
con.commit()


Write Preview
Loading…
Cancel
Save