khangtran
/
dark_web_forums
0
0
Fork 0
Code Issues 0 Pull Requests 0 Projects 0 Releases 0 Wiki Activity
Browse Source

updated markets schema

main
westernmeadow 1 year ago
parent
e0b52e0ec2
commit
98acc6f121
5 changed files with 196 additions and 223 deletions
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +0
    -14
      Forums/DB_Connection/db_connection.py
  2. +98
    -72
      MarketPlaces/DB_Connection/db_connection.py
  3. +41
    -72
      MarketPlaces/Initialization/prepare_parser.py
  4. +29
    -39
      MarketPlaces/Tor2door/parser.py
  5. +28
    -26
      MarketPlaces/Utilities/utilities.py

+ 0
- 14
Forums/DB_Connection/db_connection.py View File

@ -5,20 +5,6 @@ import traceback
import time import time
from datetime import date from datetime import date
'''
user
add
forum_id
user_reputation
status_user
interest_user
post
remove
user_reputation
status_user
interest_user
'''
def connectDataBase(): def connectDataBase():


+ 98
- 72
MarketPlaces/DB_Connection/db_connection.py View File

@ -5,24 +5,6 @@ import traceback
import time import time
from datetime import date from datetime import date
'''
vendors
add
mk_id
vendor_rating
successful_transactions
products
add
product_rating
remove
date_item
escrow_info
last_viewed
vendor_rating
successful_transaction
'''
def connectDataBase(): def connectDataBase():
@ -40,7 +22,7 @@ def verifyMarketPlace(cur, nameMarket):
try: try:
cur.execute("select id_mk from marketPlaces where name_mk = %(nameMarket)s limit 1", {'nameMarket': nameMarket})
cur.execute("select market_id from marketPlaces where name_market = %(nameMarket)s limit 1", {'nameMarket': nameMarket})
recset = cur.fetchall() recset = cur.fetchall()
@ -55,11 +37,12 @@ def verifyMarketPlace(cur, nameMarket):
print (trace) print (trace)
def verifyVendor(cur, nameVendor):
def verifyVendor(cur, nameVendor, marketId):
try: try:
cur.execute("select id_ve from vendors where name_ve = %(nameVendor)s limit 1", {'nameVendor': nameVendor})
cur.execute("select vendor_id from vendors where name_vendor = %(nameVendor)s and market_id = %(marketId)s "
"limit 1", {'nameVendor': nameVendor, 'marketId': marketId})
recset = cur.fetchall() recset = cur.fetchall()
@ -78,7 +61,7 @@ def getLastMarketPlace(cur):
try: try:
cur.execute("select id_mk from marketPlaces order by id_mk desc limit 1")
cur.execute("select market_id from marketPlaces order by market_id desc limit 1")
recset = cur.fetchall() recset = cur.fetchall()
@ -97,7 +80,7 @@ def getLastVendor(cur):
try: try:
cur.execute("select id_ve from vendors order by id_ve desc limit 1")
cur.execute("select vendor_id from vendors order by vendor_id desc limit 1")
recset = cur.fetchall() recset = cur.fetchall()
@ -114,54 +97,70 @@ def getLastVendor(cur):
def create_marketPlace(cur, row): def create_marketPlace(cur, row):
marketPlace = verifyMarketPlace(cur, row[0])
marketId = verifyMarketPlace(cur, row[0])
if not marketPlace:
marketPlace = int(getLastMarketPlace(cur) + 1)
if not marketId:
marketId = int(getLastMarketPlace(cur) + 1)
sql = "Insert into marketPlaces (id_mk, name_mk, url_mk, dateInserted) Values (%s, %s, %s, %s)"
sql = "Insert into marketplaces (market_id, name_market, url_market, dateinserted_market) " \
"Values (%s, %s, %s, %s)"
recset = [marketPlace, row[0], None, time.asctime()]
recset = [marketId, row[0], None, row[21]]
cur.execute(sql, recset) cur.execute(sql, recset)
return marketPlace
return marketId
def create_vendor(cur, row):
def create_vendor(cur, row, marketId):
vendor = verifyVendor(cur, row[18])
vendorId = verifyVendor(cur, row[1], marketId)
if not vendor:
vendor = int(getLastVendor(cur) + 1)
if not vendorId:
vendorId = int(getLastVendor(cur) + 1)
sql = "Insert into vendors (id_ve, name_ve, dateInserted) Values (%s, %s, %s)"
sql = "Insert into vendors (vendor_id, market_id, name_vendor, rating_vendor, successfultransactions_vendor, dateinserted_vendor) Values (%s, %s, %s, %s, %s, %s)"
recset = [vendor, row[18], time.asctime()]
recset = [vendorId, marketId,
row[1],
row[2] if row[2] != '-1' else None,
row[3] if row[3] != '-1' else None,
row[21]]
cur.execute(sql, recset) cur.execute(sql, recset)
return vendor
def create_items(cur, row, marketPlace, vendor):
sql = "Insert into items (id_mk, id_ve, date_item, name_item, description_item, cve_item, ms_item, category_item, " \
"escrowinfo_item, views_item, reviews_item, addeddate_item, lastvieweddate_item, btcsellingprice_item, usdsellingprice_item, " \
"eurosellingprice_item, quantitysold_item, quantityleft_item, shippedfrom_item, shippedto_item, vendorrating, successfulltransactions, " \
"termsandconditions, dateinserted_item, classification_item) Values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, " \
"%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
recset = [marketPlace, vendor, str("%02d" %date.today().month) + "/" + str("%02d" %date.today().day) + "/" + str("%04d" %date.today().year),
#recset = [marketPlace, vendor, str("%02d" %date.today().day) + "/" + str("%02d" %date.today().month) + "/" + str("%04d" %date.today().year),
row[1] if row[1]!= '-1' else None, row[5] if row[5]!= '-1' else None, row[2] if row[2]!= '-1' else None,
row[3] if row[3]!= '-1' else None, row[4] if row[4]!= '-1' else None, row[6] if row[6]!= '-1' else None,
row[7] if row[7]!= '-1' else None, row[8] if row[8]!= '-1' else None, row[9] if row[9]!= '-1' else None,
row[10] if row[10]!= '-1' else None, row[11] if row[11]!= '-1' else None, row[12] if row[12]!= '-1' else None,
row[13] if row[13]!= '-1' else None, row[14] if row[14]!= '-1' else None, row[15] if row[15]!= '-1' else None,
row[16] if row[16]!= '-1' else None, row[17] if row[17]!= '-1' else None, row[19] if row[19]!= '-1' else None,
row[20] if row[20]!= '-1' else None, row[21] if row[21]!= '-1' else None, row[22] if row[22]!= '-1' else None,
row[24] if row[24]!= '-1' else None]
return vendorId
def create_items(cur, row, marketId, vendorId):
sql = "Insert into items (market_id, vendor_id, name_item, description_item, cve_item, ms_item, category_item, " \
"views_item, reviews_item, rating_item, dateadded_item, btc_item, usd_item, euro_item, quantitysold_item, " \
"quantityleft_item, shippedfrom_item, shippedto_item, href_item, lastseen_item, dateinserted_item, " \
"classification_item) Values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, " \
"%s, %s, %s)"
recset = [marketId, vendorId,
row[4],
row[5] if row[5] != '-1' else None,
row[6] if row[6] != '-1' else None,
row[7] if row[7] != '-1' else None,
row[8] if row[8] != '-1' else None,
row[9] if row[9] != '-1' else None,
row[10] if row[10] != '-1' else None,
row[11] if row[11] != '-1' else None,
row[12] if row[12] != '-1' else None,
row[13] if row[13] != '-1' else None,
row[14] if row[14] != '-1' else None,
row[15] if row[15] != '-1' else None,
row[16] if row[16] != '-1' else None,
row[17] if row[17] != '-1' else None,
row[18] if row[18] != '-1' else None,
row[19] if row[19] != '-1' else None,
row[20] if row[20] != '-1' else None,
row[21],
row[21],
row[22]]
cur.execute(sql, recset) cur.execute(sql, recset)
@ -170,26 +169,53 @@ def create_database(cur, con):
try: try:
sql = "create table marketplaces(id_mk integer not null, name_mk character varying(255) not null, " \
"url_mk character varying(255) null, dateinserted timestamp(6) with time zone not null, " \
"constraint pk_mk primary key (id_mk))"
sql = "create table marketplaces(market_id integer not null, name_market character varying(255) not null, " \
"url_market character varying(255) null, dateinserted_market timestamp(6) with time zone not null, " \
"constraint markets_pk primary key (market_id))"
cur.execute(sql)
sql = "create table vendors(vendor_id integer not null, market_id integer not null, name_vendor character " \
"varying(255) not null, rating_vendor character varying(255), successfultransactions_vendor integer " \
"null, dateinserted_vendor timestamp(6) with time zone not null, constraint vendors_pk primary key (" \
"vendor_id), constraint vendors_market_id_fkey foreign key (market_id) references marketplaces (" \
"market_id))"
cur.execute(sql)
sql = "create table vendors_history(vendor_id integer not null, market_id integer not null, name_vendor " \
"character varying(255) not null, rating_vendor character varying(255), successfultransactions_vendor " \
"integer null, dateinserted_vendor timestamp(6) with time zone not null, constraint vendors_history_pk " \
"primary key (vendor_id, dateinserted_vendor), constraint vendors_history_vendor_id_fkey foreign key (" \
"vendor_id) references vendors (vendor_id), constraint vendors_history_market_id_fkey foreign key (" \
"market_id) references marketplaces (market_id))"
cur.execute(sql) cur.execute(sql)
sql = "create table vendors(id_ve integer not null, name_ve character varying(255) not null, " \
"dateinserted timestamp(6) with time zone not null, constraint pk_ve primary key (id_ve))"
sql = "create table items(market_id integer not null, vendor_id integer not null, name_item character " \
"varying(255) not null, description_item character varying(1000000) null, cve_item character varying(" \
"255) null, ms_item character varying(255) null, category_item character varying(255) null, views_item " \
"integer null, reviews_item integer null, rating_item character varying(255) null, dateadded_item " \
"character varying(25) null, btc_item character varying(255) null, usd_item character varying(255) " \
"null, euro_item character varying(255) null, quantitysold_item integer null, quantityleft_item " \
"character varying(255) null, shippedfrom_item character varying(255) null, shippedto_item character " \
"varying(255) null, href_item character varying(255) null, lastseen_item timestamp(6) with time zone " \
"not null, dateinserted_item timestamp(6) with time zone not null, classification_item double " \
"precision not null, constraint items_pk primary key (market_id, vendor_id, name_item), constraint " \
"items_market_id_fkey foreign key (market_id) references marketplaces (market_id),constraint " \
"items_vendor_id_fkey foreign key (vendor_id) references vendors (vendor_id))"
cur.execute(sql) cur.execute(sql)
sql = "create table items(id_mk integer not null, id_ve integer not null, date_item date not null, name_item character varying(255) not null, " \
"description_item character varying(1000000) null, cve_item character varying(255) null, ms_item character varying(255) null, " \
"category_item character varying(255) null, escrowinfo_item character varying(1000) null, views_item integer null, " \
"reviews_item character varying(255) null, addeddate_item character varying(25) null, " \
"lastvieweddate_item character varying(25) null, btcsellingprice_item character varying(255) null, " \
"usdsellingprice_item character varying(255) null, eurosellingprice_item character varying(255) null, quantitysold_item integer null, " \
"quantityleft_item character varying(255) null, shippedfrom_item character varying(255) null, shippedto_item character varying(5000) null, " \
"vendorrating character varying(255) null, successfulltransactions character varying(500) null, " \
"termsandconditions character varying(15000) null, dateinserted_item timestamp(6) with time zone not null, " \
"classification_item double precision not null, constraint pk_items primary key (id_mk, id_ve, date_item, name_item), " \
"constraint items_id_mk_fkey foreign key (id_mk) references marketplaces (id_mk),constraint items_id_ve_fkey foreign key (id_ve) references vendors (id_ve))"
sql = "create table items_history(market_id integer not null, vendor_id integer not null, name_item character " \
"varying(255) not null, description_item character varying(1000000) null, cve_item character varying(" \
"255) null, ms_item character varying(255) null, category_item character varying(255) null, views_item " \
"integer null, reviews_item integer null, rating_item character varying(255) null, dateadded_item " \
"character varying(25) null, btc_item character varying(255) null, usd_item character varying(255) " \
"null, euro_item character varying(255) null, quantitysold_item integer null, quantityleft_item " \
"character varying(255) null, shippedfrom_item character varying(255) null, shippedto_item character " \
"varying(255) null, href_item character varying(255) null, lastseen_item timestamp(6) with time zone " \
"not null, dateinserted_item timestamp(6) with time zone not null, classification_item double " \
"precision not null, constraint items_history_pk primary key (market_id, vendor_id, name_item, " \
"dateinserted_item), constraint items_history_market_id_fkey foreign key (market_id) references " \
"marketplaces (market_id), constraint items_history_vendor_id_fkey foreign key (vendor_id) references " \
"vendors (vendor_id))"
cur.execute(sql) cur.execute(sql)
con.commit() con.commit()


+ 41
- 72
MarketPlaces/Initialization/prepare_parser.py View File

@ -18,75 +18,44 @@ def mergePages(rmm, rec):
print("----------------- Matched: " + rec[1] + "--------------------") print("----------------- Matched: " + rec[1] + "--------------------")
# if rec[1] == "-1": #Item_Name
# rec[1] = rmm[0]
rec[1] = rmm[0]
if rec[2] == "-1": #Item_CVE_Classification
rec[2] = rmm[4]
if rec[3] == "-1": #Item_MS_Classification
rec[3] = rmm[5]
if rec[4] == "-1": #Item_MarketCategory
rec[4] = rmm[7]
if rec[5] == "-1": #Item_Description
rec[5] = rmm[1]
elif rmm[1] != "-1":
rec[5] = rec[5] + " " + rmm[1]
if rec[6] == "-1": #Item _EscrowInfo
rec[6] = rmm[11]
#rec[7] = "-1" #Item__N.OfViews
if rec[8] == "-1": #Item_Reviews
rec[8] = rmm[6]
if rec[9] == "-1": #Item_AddedDate
rec[9] = rmm[15]
if rec[10] == "-1": #Item_LastViewedDate
rec[10] = rmm[2]
if rec[11] == "-1": #Item_BTC_SellingPrice
rec[11] = rmm[18]
if rec[12] == "-1": #Item_US_SellingPrice
rec[12] = rmm[19]
if rec[13] == "-1": #Item_EURO_SellingPrice
rec[13] = rmm[22]
if rec[14] == "-1": #Item_QuantitySold
rec[14] = rmm[14]
if rec[15] == "-1": #Item_QuantityLeft
rec[15] = rmm[10]
if rec[16] == "-1": #Item_ShippedFrom
rec[16] = rmm[8]
if rec[17] == "-1": #Item_ShippedTo
rec[17] = rmm[9]
if rec[18] == "-1": #Vendor_Name
rec[18] = rmm[13]
if rec[19] == "-1": #Vendor_Rating
rec[19] = rmm[20]
if rec[20] == "-1": #Vendor_Successfull Transactions
rec[20] = rmm[21]
if rec[21] == "-1": #Vendor_TermsAndConditions
rec[21] = rmm[12]
#rec[?] = rmm[17] #Item_EndDate
#rec[?] = rmm[?] #Item_Feedback
#rec[?] = rmm[?] #Shipping Options
#rec[?] = rmm[?] #Average Delivery Time
if rec[1] == "-1": # name_vendor
rec[1] = rmm[0]
if rec[2] == "-1": # rating_vendor
rec[2] = rmm[1]
if rec[3] == "-1": # success_vendor
rec[3] = rmm[2]
if rec[4] == "-1": # name_item
rec[4] = rmm[3]
if rec[5] == "-1": # description_item
rec[5] = rmm[4]
if rec[6] == "-1": # cve_item
rec[6] = rmm[5]
if rec[7] == "-1": # ms_item
rec[7] = rmm[6]
if rec[8] == "-1": # category_item
rec[8] = rmm[7]
if rec[9] == "-1": # views_item
rec[9] = rmm[8]
if rec[10] == "-1": # reviews_item
rec[10] = rmm[9]
if rec[11] == "-1": # rating_item
rec[11] = rmm[10]
if rec[12] == "-1": # adddate_item
rec[12] = rmm[11]
if rec[13] == "-1": # btc_item
rec[13] = rmm[12]
if rec[14] == "-1": # usd_item
rec[14] = rmm[13]
if rec[15] == "-1": # euro_item
rec[15] = rmm[14]
if rec[16] == "-1": # quantitysold_item
rec[16] = rmm[15]
if rec[17] == "-1": # quantityleft_item
rec[17] = rmm[16]
if rec[18] == "-1": # shippedfrom_item
rec[18] = rmm[17]
if rec[19] == "-1": # shippedto_item
rec[19] = rmm[18]
return rec return rec
@ -95,7 +64,7 @@ def persist_data(row, cur):
marketPlace = create_marketPlace(cur, row) marketPlace = create_marketPlace(cur, row)
vendor = create_vendor(cur, row)
vendor = create_vendor(cur, row, marketPlace)
create_items(cur, row, marketPlace, vendor) create_items(cur, row, marketPlace, vendor)
@ -246,7 +215,7 @@ def new_parse(marketPlace, createLog):
# key = u"Pr:" + rec[1].upper()[:list_lim1] + u" Vendor:" + rec[18].upper()[:list_lim2] # key = u"Pr:" + rec[1].upper()[:list_lim1] + u" Vendor:" + rec[18].upper()[:list_lim2]
# key = u"Pr:" + rec[1].upper() # key = u"Pr:" + rec[1].upper()
url = ''.join(e for e in rec[23] if e.isalnum())
url = ''.join(e for e in rec[20] if e.isalnum())
key = u"Url:" + url key = u"Url:" + url
# if the associated description page is parsed # if the associated description page is parsed
@ -260,7 +229,7 @@ def new_parse(marketPlace, createLog):
# Append to the list the classification of the product # Append to the list the classification of the product
# rec.append(str(predict(rec[1], rec[5], language='markets'))) # rec.append(str(predict(rec[1], rec[5], language='markets')))
rec.append(str(predict(rec[1], rec[5], language='sup_english')))
rec.append(str(predict(rec[4], rec[5], language='sup_english')))
# Persisting the information in the database # Persisting the information in the database
try: try:


+ 29
- 39
MarketPlaces/Tor2door/parser.py View File

@ -12,29 +12,25 @@ def tor2door_description_parser(soup):
# Fields to be parsed # Fields to be parsed
name = "-1" # 0 Product_Name y
describe = "-1" # 1 Product_Description y
lastSeen = "-1" # 2 Product_LastViewDate
rules = "-1" # 3 NOT USED ...
CVE = "-1" # 4 Product_CVE_Classification (Common Vulnerabilities and Exposures)
MS = "-1" # 5 Product_MS_Classification (Microsoft Security)
review = "-1" # 6 Product_Number_Of_Reviews
vendor = "-1" # 0 Vendor_Name
success = "-1" # 1 Vendor_Successful_Transactions
rating_vendor = "-1" # 2 Vendor_Rating
name = "-1" # 3 Product_Name
describe = "-1" # 4 Product_Description
CVE = "-1" # 5 Product_CVE_Classification (Common Vulnerabilities and Exposures)
MS = "-1" # 6 Product_MS_Classification (Microsoft Security)
category = "-1" # 7 Product_Category category = "-1" # 7 Product_Category
shipFrom = "-1" # 8 Product_ShippedFrom
shipTo = "-1" # 9 Product_ShippedTo
left = "-1" # 10 Product_QuantityLeft y
escrow = "-1" # 11 Vendor_Warranty y
terms = "-1" # 12 Vendor_TermsAndConditions
vendor = "-1" # 13 Vendor_Name y
sold = "-1" # 14 Product_QuantitySold y
addDate = "-1" # 15 Product_AddedDate
available = "-1" # 16 NOT USED ...
endDate = "-1" # 17 NOT USED ...
BTC = "-1" # 18 Product_BTC_SellingPrice y
USD = "-1" # 19 Product_USD_SellingPrice y
rating = "-1" # 20 Vendor_Rating
success = "-1" # 21 Vendor_Successful_Transactions
EURO = "-1" # 22 Product_EURO_SellingPrice
views = "-1" # 8 Product_Number_Of_Views
reviews = "-1" # 9 Product_Number_Of_Reviews
rating_item = "-1" # 10 Product_Rating
addDate = "-1" # 11 Product_AddedDate
BTC = "-1" # 12 Product_BTC_SellingPrice
USD = "-1" # 13 Product_USD_SellingPrice
EURO = "-1" # 14 Product_EURO_SellingPrice
sold = "-1" # 15 Product_QuantitySold
left = "-1" # 16 Product_QuantityLeft
shipFrom = "-1" # 17 Product_ShippedFrom
shipTo = "-1" # 18 Product_ShippedTo
bae = soup.find('div', {'class': "col-9"}) bae = soup.find('div', {'class': "col-9"})
@ -57,11 +53,6 @@ def tor2door_description_parser(soup):
# half_star = bae[2].find('i', {'class': "fas fa-star-half-alt"}) # half_star = bae[2].find('i', {'class': "fas fa-star-half-alt"})
# rating = len(full_stars) + (0.5 if half_star is not None else 0) # rating = len(full_stars) + (0.5 if half_star is not None else 0)
# Finding Warranty
escrow = mb[2].text
escrow = escrow.replace("Payment:", "")
escrow = escrow.strip()
# Finding Quantity Sold and Left # Finding Quantity Sold and Left
temp = mb[4].text.split(',') temp = mb[4].text.split(',')
@ -116,8 +107,8 @@ def tor2door_description_parser(soup):
MS = MS.replace('\n', '') MS = MS.replace('\n', '')
# Populating the final variable (this should be a list with all fields scraped) # Populating the final variable (this should be a list with all fields scraped)
row = (name, describe, lastSeen, rules, CVE, MS, review, category, shipFrom, shipTo, left, escrow, terms, vendor,
sold, addDate, available, endDate, BTC, USD, rating, success, EURO)
row = (vendor, rating_vendor, success, name, describe, CVE, MS, category, views, reviews, rating_item, addDate,
BTC, USD, EURO, sold, left, shipFrom, shipTo)
# Sending the results # Sending the results
return row return row
@ -128,17 +119,19 @@ def tor2door_listing_parser(soup):
# Fields to be parsed # Fields to be parsed
nm = 0 # Total_Products (Should be Integer) nm = 0 # Total_Products (Should be Integer)
mktName = "Tor2door" # 0 Marketplace_Name
mktName = "Tor2door" # 0 Marketplace_Name
vendor = [] # 18 Vendor y
rating_vendor = [] # 19 Vendor_Rating
success = [] # 20 Vendor_Successful_Transactions
name = [] # 1 Product_Name y name = [] # 1 Product_Name y
CVE = [] # 2 Product_CVE_Classification (Common Vulnerabilities and Exposures) CVE = [] # 2 Product_CVE_Classification (Common Vulnerabilities and Exposures)
MS = [] # 3 Product_MS_Classification (Microsoft Security) MS = [] # 3 Product_MS_Classification (Microsoft Security)
category = [] # 4 Product_Category y category = [] # 4 Product_Category y
describe = [] # 5 Product_Description describe = [] # 5 Product_Description
escrow = [] # 6 Vendor_Warranty
views = [] # 7 Product_Number_Of_Views views = [] # 7 Product_Number_Of_Views
reviews = [] # 8 Product_Number_Of_Reviews y
reviews = [] # 7 Product_Number_Of_Reviews
rating_item = [] # 8 Product_Rating
addDate = [] # 9 Product_AddDate addDate = [] # 9 Product_AddDate
lastSeen = [] # 10 Product_LastViewDate
BTC = [] # 11 Product_BTC_SellingPrice BTC = [] # 11 Product_BTC_SellingPrice
USD = [] # 12 Product_USD_SellingPrice y USD = [] # 12 Product_USD_SellingPrice y
EURO = [] # 13 Product_EURO_SellingPrice EURO = [] # 13 Product_EURO_SellingPrice
@ -146,10 +139,7 @@ def tor2door_listing_parser(soup):
qLeft =[] # 15 Product_QuantityLeft qLeft =[] # 15 Product_QuantityLeft
shipFrom = [] # 16 Product_ShippedFrom shipFrom = [] # 16 Product_ShippedFrom
shipTo = [] # 17 Product_ShippedTo shipTo = [] # 17 Product_ShippedTo
vendor = [] # 18 Vendor y
rating = [] # 19 Vendor_Rating
success = [] # 20 Vendor_Successful_Transactions
href = [] # 24 Product_Links (Urls)
href = [] # 24 Product_Links
listing = soup.findAll('div', {"class": "card product-card mb-3"}) listing = soup.findAll('div', {"class": "card product-card mb-3"})
@ -228,8 +218,8 @@ def tor2door_listing_parser(soup):
MS.append(MSValue) MS.append(MSValue)
# Populate the final variable (this should be a list with all fields scraped) # Populate the final variable (this should be a list with all fields scraped)
return organizeProducts(mktName, nm, name, CVE, MS, category, describe, escrow, views, reviews, addDate, lastSeen,
BTC, USD, EURO, qLeft, shipFrom, shipTo, vendor, rating, success, sold, href)
return organizeProducts(mktName, nm, vendor, rating_vendor, success, name, CVE, MS, category, describe, views,
reviews, rating_item, addDate, BTC, USD, EURO, sold, qLeft, shipFrom, shipTo, href)
def tor2door_links_parser(soup): def tor2door_links_parser(soup):


+ 28
- 26
MarketPlaces/Utilities/utilities.py View File

@ -207,57 +207,59 @@ def cleanLink(originalLink):
return originalLink return originalLink
def organizeProducts(marketplace, nm, nombre, CVE, MS, category, describe, escrow, views, reviews, addDate, lastSeen,
BTC, USD, EURO, qLeft, shipFrom, shipTo, user, rating, success, sold, href):
def organizeProducts(marketplace, nm, vendor, rating_vendor, success_vendor, nombre, CVE, MS, category, describe,
views, reviews, rating_item, addDate, BTC, USD, EURO, sold, qLeft, shipFrom, shipTo, href):
rw = [] rw = []
day = time.strftime("%m/%d/%Y") day = time.strftime("%m/%d/%Y")
#day = time.strftime("%d/%m/%Y")
ahora = time.strftime("%I:%M:%S") ahora = time.strftime("%I:%M:%S")
for n in range(nm): for n in range(nm):
lne = marketplace + "," #0
lne += "-1" if len(nombre) == 0 else nombre[n] #1
lne = marketplace # 0
lne += ","
lne += vendor[n] # 1
lne += ","
lne += "-1" if len(rating_vendor) == 0 else rating_vendor[n] # 2
lne += ","
lne += "-1" if len(success_vendor) == 0 else success_vendor[n] # 3
lne += ","
lne += nombre[n] # 4
lne += ',' lne += ','
lne += "-1" if len(CVE) == 0 else CVE[n] #2
lne += "-1" if len(describe) == 0 else describe[n] # 5
lne += "," lne += ","
lne += "-1" if len(MS) == 0 else MS[n] #3
lne += "-1" if len(CVE) == 0 else CVE[n] # 6
lne += "," lne += ","
lne += "-1" if len(category) == 0 else category[n] #4
lne += "-1" if len(MS) == 0 else MS[n] # 7
lne += "," lne += ","
lne += "-1" if len(describe) == 0 else describe[n] #5
lne += "-1" if len(category) == 0 else category[n] # 8
lne += "," lne += ","
lne += "-1" if len(escrow) == 0 else escrow[n] #6
lne += "-1" if len(views) == 0 else views[n] # 9
lne += "," lne += ","
lne += "-1" if len(views) == 0 else views[n] #7
lne += "-1" if len(reviews) == 0 else reviews[n] # 10
lne += "," lne += ","
lne += "-1" if len(reviews) == 0 else reviews[n] #8
lne += "-1" if len(rating_item) == 0 else rating_item[n] # 11
lne += "," lne += ","
lne += "-1" if len(addDate) == 0 else addDate[n] #9
lne += "-1" if len(addDate) == 0 else addDate[n] # 12
lne += "," lne += ","
lne += "-1" if len(lastSeen) == 0 else lastSeen[n] #10
lne += "-1" if len(BTC) == 0 else BTC[n] # 13
lne += "," lne += ","
lne += "-1" if len(BTC) == 0 else BTC[n] #11
lne += "-1" if len(USD) == 0 else USD[n] # 14
lne += "," lne += ","
lne += "-1" if len(USD) == 0 else USD[n] #12
lne += "-1" if len(EURO) == 0 else EURO[n] # 15
lne += "," lne += ","
lne += "-1" if len(EURO) == 0 else EURO[n] #13
lne += "-1" if len(sold) == 0 else sold[n] # 16
lne += "," lne += ","
lne += "-1" if len(sold) == 0 else sold[n] #14
lne += "-1" if len(qLeft) == 0 else qLeft[n] # 17
lne += "," lne += ","
lne += "-1" if len(qLeft) == 0 else qLeft[n] #15
lne += "-1" if len(shipFrom) == 0 else shipFrom[n] # 18
lne += "," lne += ","
lne += "-1" if len(shipFrom) == 0 else shipFrom[n] #16
lne += "-1" if len(shipTo) == 0 else shipTo[n] # 19
lne += "," lne += ","
lne += "-1" if len(shipTo) == 0 else shipTo[n] #17
lne += "," + user[n] + "," #18
lne += "-1" if len(rating) == 0 else rating[n] #19
lne += "-1" if len(href) == 0 else href[n] # 20
lne += "," lne += ","
lne += "-1" if len(success) == 0 else success[n] #20
lne += "," + "-1" + "," + day + " " + ahora + "," #21, 22
lne += "-1" if len(href) == 0 else href[n] #23
lne += day + " " + ahora # 21
rw.append(lne) rw.append(lne)


Write Preview
Loading…
Cancel
Save