khangtran
/
dark_web_forums
0
0
Fork 0
Code Issues 0 Pull Requests 0 Projects 0 Releases 0 Wiki Activity
Browse Source

updated markets schema

main
westernmeadow 2 years ago
parent
e0b52e0ec2
commit
98acc6f121
5 changed files with 196 additions and 223 deletions
Unified View
  1. +0
    -14
      Forums/DB_Connection/db_connection.py
  2. +98
    -72
      MarketPlaces/DB_Connection/db_connection.py
  3. +41
    -72
      MarketPlaces/Initialization/prepare_parser.py
  4. +29
    -39
      MarketPlaces/Tor2door/parser.py
  5. +28
    -26
      MarketPlaces/Utilities/utilities.py

+ 0
- 14
Forums/DB_Connection/db_connection.py View File

@ -5,20 +5,6 @@ import traceback
import time import time
from datetime import date from datetime import date
'''
user
add
forum_id
user_reputation
status_user
interest_user
post
remove
user_reputation
status_user
interest_user
'''
def connectDataBase(): def connectDataBase():


+ 98
- 72
MarketPlaces/DB_Connection/db_connection.py View File

@ -5,24 +5,6 @@ import traceback
import time import time
from datetime import date from datetime import date
'''
vendors
add
mk_id
vendor_rating
successful_transactions
products
add
product_rating
remove
date_item
escrow_info
last_viewed
vendor_rating
successful_transaction
'''
def connectDataBase(): def connectDataBase():
@ -40,7 +22,7 @@ def verifyMarketPlace(cur, nameMarket):
try: try:
cur.execute("select id_mk from marketPlaces where name_mk = %(nameMarket)s limit 1", {'nameMarket': nameMarket}) cur.execute("select market_id from marketPlaces where name_market = %(nameMarket)s limit 1", {'nameMarket': nameMarket})
recset = cur.fetchall() recset = cur.fetchall()
@ -55,11 +37,12 @@ def verifyMarketPlace(cur, nameMarket):
print (trace) print (trace)
def verifyVendor(cur, nameVendor): def verifyVendor(cur, nameVendor, marketId):
try: try:
cur.execute("select id_ve from vendors where name_ve = %(nameVendor)s limit 1", {'nameVendor': nameVendor}) cur.execute("select vendor_id from vendors where name_vendor = %(nameVendor)s and market_id = %(marketId)s "
"limit 1", {'nameVendor': nameVendor, 'marketId': marketId})
recset = cur.fetchall() recset = cur.fetchall()
@ -78,7 +61,7 @@ def getLastMarketPlace(cur):
try: try:
cur.execute("select id_mk from marketPlaces order by id_mk desc limit 1") cur.execute("select market_id from marketPlaces order by market_id desc limit 1")
recset = cur.fetchall() recset = cur.fetchall()
@ -97,7 +80,7 @@ def getLastVendor(cur):
try: try:
cur.execute("select id_ve from vendors order by id_ve desc limit 1") cur.execute("select vendor_id from vendors order by vendor_id desc limit 1")
recset = cur.fetchall() recset = cur.fetchall()
@ -114,54 +97,70 @@ def getLastVendor(cur):
def create_marketPlace(cur, row): def create_marketPlace(cur, row):
marketPlace = verifyMarketPlace(cur, row[0]) marketId = verifyMarketPlace(cur, row[0])
if not marketPlace: if not marketId:
marketPlace = int(getLastMarketPlace(cur) + 1) marketId = int(getLastMarketPlace(cur) + 1)
sql = "Insert into marketPlaces (id_mk, name_mk, url_mk, dateInserted) Values (%s, %s, %s, %s)" sql = "Insert into marketplaces (market_id, name_market, url_market, dateinserted_market) " \
"Values (%s, %s, %s, %s)"
recset = [marketPlace, row[0], None, time.asctime()] recset = [marketId, row[0], None, row[21]]
cur.execute(sql, recset) cur.execute(sql, recset)
return marketPlace return marketId
def create_vendor(cur, row): def create_vendor(cur, row, marketId):
vendor = verifyVendor(cur, row[18]) vendorId = verifyVendor(cur, row[1], marketId)
if not vendor: if not vendorId:
vendor = int(getLastVendor(cur) + 1) vendorId = int(getLastVendor(cur) + 1)
sql = "Insert into vendors (id_ve, name_ve, dateInserted) Values (%s, %s, %s)" sql = "Insert into vendors (vendor_id, market_id, name_vendor, rating_vendor, successfultransactions_vendor, dateinserted_vendor) Values (%s, %s, %s, %s, %s, %s)"
recset = [vendor, row[18], time.asctime()] recset = [vendorId, marketId,
row[1],
row[2] if row[2] != '-1' else None,
row[3] if row[3] != '-1' else None,
row[21]]
cur.execute(sql, recset) cur.execute(sql, recset)
return vendor return vendorId
def create_items(cur, row, marketId, vendorId):
sql = "Insert into items (market_id, vendor_id, name_item, description_item, cve_item, ms_item, category_item, " \
def create_items(cur, row, marketPlace, vendor): "views_item, reviews_item, rating_item, dateadded_item, btc_item, usd_item, euro_item, quantitysold_item, " \
"quantityleft_item, shippedfrom_item, shippedto_item, href_item, lastseen_item, dateinserted_item, " \
sql = "Insert into items (id_mk, id_ve, date_item, name_item, description_item, cve_item, ms_item, category_item, " \ "classification_item) Values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, " \
"escrowinfo_item, views_item, reviews_item, addeddate_item, lastvieweddate_item, btcsellingprice_item, usdsellingprice_item, " \ "%s, %s, %s)"
"eurosellingprice_item, quantitysold_item, quantityleft_item, shippedfrom_item, shippedto_item, vendorrating, successfulltransactions, " \ recset = [marketId, vendorId,
"termsandconditions, dateinserted_item, classification_item) Values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, " \ row[4],
"%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)" row[5] if row[5] != '-1' else None,
row[6] if row[6] != '-1' else None,
recset = [marketPlace, vendor, str("%02d" %date.today().month) + "/" + str("%02d" %date.today().day) + "/" + str("%04d" %date.today().year), row[7] if row[7] != '-1' else None,
#recset = [marketPlace, vendor, str("%02d" %date.today().day) + "/" + str("%02d" %date.today().month) + "/" + str("%04d" %date.today().year), row[8] if row[8] != '-1' else None,
row[1] if row[1]!= '-1' else None, row[5] if row[5]!= '-1' else None, row[2] if row[2]!= '-1' else None, row[9] if row[9] != '-1' else None,
row[3] if row[3]!= '-1' else None, row[4] if row[4]!= '-1' else None, row[6] if row[6]!= '-1' else None, row[10] if row[10] != '-1' else None,
row[7] if row[7]!= '-1' else None, row[8] if row[8]!= '-1' else None, row[9] if row[9]!= '-1' else None, row[11] if row[11] != '-1' else None,
row[10] if row[10]!= '-1' else None, row[11] if row[11]!= '-1' else None, row[12] if row[12]!= '-1' else None, row[12] if row[12] != '-1' else None,
row[13] if row[13]!= '-1' else None, row[14] if row[14]!= '-1' else None, row[15] if row[15]!= '-1' else None, row[13] if row[13] != '-1' else None,
row[16] if row[16]!= '-1' else None, row[17] if row[17]!= '-1' else None, row[19] if row[19]!= '-1' else None, row[14] if row[14] != '-1' else None,
row[20] if row[20]!= '-1' else None, row[21] if row[21]!= '-1' else None, row[22] if row[22]!= '-1' else None, row[15] if row[15] != '-1' else None,
row[24] if row[24]!= '-1' else None] row[16] if row[16] != '-1' else None,
row[17] if row[17] != '-1' else None,
row[18] if row[18] != '-1' else None,
row[19] if row[19] != '-1' else None,
row[20] if row[20] != '-1' else None,
row[21],
row[21],
row[22]]
cur.execute(sql, recset) cur.execute(sql, recset)
@ -170,26 +169,53 @@ def create_database(cur, con):
try: try:
sql = "create table marketplaces(id_mk integer not null, name_mk character varying(255) not null, " \ sql = "create table marketplaces(market_id integer not null, name_market character varying(255) not null, " \
"url_mk character varying(255) null, dateinserted timestamp(6) with time zone not null, " \ "url_market character varying(255) null, dateinserted_market timestamp(6) with time zone not null, " \
"constraint pk_mk primary key (id_mk))" "constraint markets_pk primary key (market_id))"
cur.execute(sql)
sql = "create table vendors(vendor_id integer not null, market_id integer not null, name_vendor character " \
"varying(255) not null, rating_vendor character varying(255), successfultransactions_vendor integer " \
"null, dateinserted_vendor timestamp(6) with time zone not null, constraint vendors_pk primary key (" \
"vendor_id), constraint vendors_market_id_fkey foreign key (market_id) references marketplaces (" \
"market_id))"
cur.execute(sql)
sql = "create table vendors_history(vendor_id integer not null, market_id integer not null, name_vendor " \
"character varying(255) not null, rating_vendor character varying(255), successfultransactions_vendor " \
"integer null, dateinserted_vendor timestamp(6) with time zone not null, constraint vendors_history_pk " \
"primary key (vendor_id, dateinserted_vendor), constraint vendors_history_vendor_id_fkey foreign key (" \
"vendor_id) references vendors (vendor_id), constraint vendors_history_market_id_fkey foreign key (" \
"market_id) references marketplaces (market_id))"
cur.execute(sql) cur.execute(sql)
sql = "create table vendors(id_ve integer not null, name_ve character varying(255) not null, " \ sql = "create table items(market_id integer not null, vendor_id integer not null, name_item character " \
"dateinserted timestamp(6) with time zone not null, constraint pk_ve primary key (id_ve))" "varying(255) not null, description_item character varying(1000000) null, cve_item character varying(" \
"255) null, ms_item character varying(255) null, category_item character varying(255) null, views_item " \
"integer null, reviews_item integer null, rating_item character varying(255) null, dateadded_item " \
"character varying(25) null, btc_item character varying(255) null, usd_item character varying(255) " \
"null, euro_item character varying(255) null, quantitysold_item integer null, quantityleft_item " \
"character varying(255) null, shippedfrom_item character varying(255) null, shippedto_item character " \
"varying(255) null, href_item character varying(255) null, lastseen_item timestamp(6) with time zone " \
"not null, dateinserted_item timestamp(6) with time zone not null, classification_item double " \
"precision not null, constraint items_pk primary key (market_id, vendor_id, name_item), constraint " \
"items_market_id_fkey foreign key (market_id) references marketplaces (market_id),constraint " \
"items_vendor_id_fkey foreign key (vendor_id) references vendors (vendor_id))"
cur.execute(sql) cur.execute(sql)
sql = "create table items(id_mk integer not null, id_ve integer not null, date_item date not null, name_item character varying(255) not null, " \ sql = "create table items_history(market_id integer not null, vendor_id integer not null, name_item character " \
"description_item character varying(1000000) null, cve_item character varying(255) null, ms_item character varying(255) null, " \ "varying(255) not null, description_item character varying(1000000) null, cve_item character varying(" \
"category_item character varying(255) null, escrowinfo_item character varying(1000) null, views_item integer null, " \ "255) null, ms_item character varying(255) null, category_item character varying(255) null, views_item " \
"reviews_item character varying(255) null, addeddate_item character varying(25) null, " \ "integer null, reviews_item integer null, rating_item character varying(255) null, dateadded_item " \
"lastvieweddate_item character varying(25) null, btcsellingprice_item character varying(255) null, " \ "character varying(25) null, btc_item character varying(255) null, usd_item character varying(255) " \
"usdsellingprice_item character varying(255) null, eurosellingprice_item character varying(255) null, quantitysold_item integer null, " \ "null, euro_item character varying(255) null, quantitysold_item integer null, quantityleft_item " \
"quantityleft_item character varying(255) null, shippedfrom_item character varying(255) null, shippedto_item character varying(5000) null, " \ "character varying(255) null, shippedfrom_item character varying(255) null, shippedto_item character " \
"vendorrating character varying(255) null, successfulltransactions character varying(500) null, " \ "varying(255) null, href_item character varying(255) null, lastseen_item timestamp(6) with time zone " \
"termsandconditions character varying(15000) null, dateinserted_item timestamp(6) with time zone not null, " \ "not null, dateinserted_item timestamp(6) with time zone not null, classification_item double " \
"classification_item double precision not null, constraint pk_items primary key (id_mk, id_ve, date_item, name_item), " \ "precision not null, constraint items_history_pk primary key (market_id, vendor_id, name_item, " \
"constraint items_id_mk_fkey foreign key (id_mk) references marketplaces (id_mk),constraint items_id_ve_fkey foreign key (id_ve) references vendors (id_ve))" "dateinserted_item), constraint items_history_market_id_fkey foreign key (market_id) references " \
"marketplaces (market_id), constraint items_history_vendor_id_fkey foreign key (vendor_id) references " \
"vendors (vendor_id))"
cur.execute(sql) cur.execute(sql)
con.commit() con.commit()


+ 41
- 72
MarketPlaces/Initialization/prepare_parser.py View File

@ -18,75 +18,44 @@ def mergePages(rmm, rec):
print("----------------- Matched: " + rec[1] + "--------------------") print("----------------- Matched: " + rec[1] + "--------------------")
# if rec[1] == "-1": #Item_Name if rec[1] == "-1": # name_vendor
# rec[1] = rmm[0] rec[1] = rmm[0]
rec[1] = rmm[0] if rec[2] == "-1": # rating_vendor
rec[2] = rmm[1]
if rec[2] == "-1": #Item_CVE_Classification if rec[3] == "-1": # success_vendor
rec[2] = rmm[4] rec[3] = rmm[2]
if rec[4] == "-1": # name_item
if rec[3] == "-1": #Item_MS_Classification rec[4] = rmm[3]
rec[3] = rmm[5] if rec[5] == "-1": # description_item
rec[5] = rmm[4]
if rec[4] == "-1": #Item_MarketCategory if rec[6] == "-1": # cve_item
rec[4] = rmm[7] rec[6] = rmm[5]
if rec[7] == "-1": # ms_item
if rec[5] == "-1": #Item_Description rec[7] = rmm[6]
rec[5] = rmm[1] if rec[8] == "-1": # category_item
elif rmm[1] != "-1": rec[8] = rmm[7]
rec[5] = rec[5] + " " + rmm[1] if rec[9] == "-1": # views_item
rec[9] = rmm[8]
if rec[6] == "-1": #Item _EscrowInfo if rec[10] == "-1": # reviews_item
rec[6] = rmm[11] rec[10] = rmm[9]
if rec[11] == "-1": # rating_item
#rec[7] = "-1" #Item__N.OfViews rec[11] = rmm[10]
if rec[12] == "-1": # adddate_item
if rec[8] == "-1": #Item_Reviews rec[12] = rmm[11]
rec[8] = rmm[6] if rec[13] == "-1": # btc_item
rec[13] = rmm[12]
if rec[9] == "-1": #Item_AddedDate if rec[14] == "-1": # usd_item
rec[9] = rmm[15] rec[14] = rmm[13]
if rec[15] == "-1": # euro_item
if rec[10] == "-1": #Item_LastViewedDate rec[15] = rmm[14]
rec[10] = rmm[2] if rec[16] == "-1": # quantitysold_item
rec[16] = rmm[15]
if rec[11] == "-1": #Item_BTC_SellingPrice if rec[17] == "-1": # quantityleft_item
rec[11] = rmm[18] rec[17] = rmm[16]
if rec[18] == "-1": # shippedfrom_item
if rec[12] == "-1": #Item_US_SellingPrice rec[18] = rmm[17]
rec[12] = rmm[19] if rec[19] == "-1": # shippedto_item
rec[19] = rmm[18]
if rec[13] == "-1": #Item_EURO_SellingPrice
rec[13] = rmm[22]
if rec[14] == "-1": #Item_QuantitySold
rec[14] = rmm[14]
if rec[15] == "-1": #Item_QuantityLeft
rec[15] = rmm[10]
if rec[16] == "-1": #Item_ShippedFrom
rec[16] = rmm[8]
if rec[17] == "-1": #Item_ShippedTo
rec[17] = rmm[9]
if rec[18] == "-1": #Vendor_Name
rec[18] = rmm[13]
if rec[19] == "-1": #Vendor_Rating
rec[19] = rmm[20]
if rec[20] == "-1": #Vendor_Successfull Transactions
rec[20] = rmm[21]
if rec[21] == "-1": #Vendor_TermsAndConditions
rec[21] = rmm[12]
#rec[?] = rmm[17] #Item_EndDate
#rec[?] = rmm[?] #Item_Feedback
#rec[?] = rmm[?] #Shipping Options
#rec[?] = rmm[?] #Average Delivery Time
return rec return rec
@ -95,7 +64,7 @@ def persist_data(row, cur):
marketPlace = create_marketPlace(cur, row) marketPlace = create_marketPlace(cur, row)
vendor = create_vendor(cur, row) vendor = create_vendor(cur, row, marketPlace)
create_items(cur, row, marketPlace, vendor) create_items(cur, row, marketPlace, vendor)
@ -246,7 +215,7 @@ def new_parse(marketPlace, createLog):
# key = u"Pr:" + rec[1].upper()[:list_lim1] + u" Vendor:" + rec[18].upper()[:list_lim2] # key = u"Pr:" + rec[1].upper()[:list_lim1] + u" Vendor:" + rec[18].upper()[:list_lim2]
# key = u"Pr:" + rec[1].upper() # key = u"Pr:" + rec[1].upper()
url = ''.join(e for e in rec[23] if e.isalnum()) url = ''.join(e for e in rec[20] if e.isalnum())
key = u"Url:" + url key = u"Url:" + url
# if the associated description page is parsed # if the associated description page is parsed
@ -260,7 +229,7 @@ def new_parse(marketPlace, createLog):
# Append to the list the classification of the product # Append to the list the classification of the product
# rec.append(str(predict(rec[1], rec[5], language='markets'))) # rec.append(str(predict(rec[1], rec[5], language='markets')))
rec.append(str(predict(rec[1], rec[5], language='sup_english'))) rec.append(str(predict(rec[4], rec[5], language='sup_english')))
# Persisting the information in the database # Persisting the information in the database
try: try:


+ 29
- 39
MarketPlaces/Tor2door/parser.py View File

@ -12,29 +12,25 @@ def tor2door_description_parser(soup):
# Fields to be parsed # Fields to be parsed
name = "-1" # 0 Product_Name y vendor = "-1" # 0 Vendor_Name
describe = "-1" # 1 Product_Description y success = "-1" # 1 Vendor_Successful_Transactions
lastSeen = "-1" # 2 Product_LastViewDate rating_vendor = "-1" # 2 Vendor_Rating
rules = "-1" # 3 NOT USED ... name = "-1" # 3 Product_Name
CVE = "-1" # 4 Product_CVE_Classification (Common Vulnerabilities and Exposures) describe = "-1" # 4 Product_Description
MS = "-1" # 5 Product_MS_Classification (Microsoft Security) CVE = "-1" # 5 Product_CVE_Classification (Common Vulnerabilities and Exposures)
review = "-1" # 6 Product_Number_Of_Reviews MS = "-1" # 6 Product_MS_Classification (Microsoft Security)
category = "-1" # 7 Product_Category category = "-1" # 7 Product_Category
shipFrom = "-1" # 8 Product_ShippedFrom views = "-1" # 8 Product_Number_Of_Views
shipTo = "-1" # 9 Product_ShippedTo reviews = "-1" # 9 Product_Number_Of_Reviews
left = "-1" # 10 Product_QuantityLeft y rating_item = "-1" # 10 Product_Rating
escrow = "-1" # 11 Vendor_Warranty y addDate = "-1" # 11 Product_AddedDate
terms = "-1" # 12 Vendor_TermsAndConditions BTC = "-1" # 12 Product_BTC_SellingPrice
vendor = "-1" # 13 Vendor_Name y USD = "-1" # 13 Product_USD_SellingPrice
sold = "-1" # 14 Product_QuantitySold y EURO = "-1" # 14 Product_EURO_SellingPrice
addDate = "-1" # 15 Product_AddedDate sold = "-1" # 15 Product_QuantitySold
available = "-1" # 16 NOT USED ... left = "-1" # 16 Product_QuantityLeft
endDate = "-1" # 17 NOT USED ... shipFrom = "-1" # 17 Product_ShippedFrom
BTC = "-1" # 18 Product_BTC_SellingPrice y shipTo = "-1" # 18 Product_ShippedTo
USD = "-1" # 19 Product_USD_SellingPrice y
rating = "-1" # 20 Vendor_Rating
success = "-1" # 21 Vendor_Successful_Transactions
EURO = "-1" # 22 Product_EURO_SellingPrice
bae = soup.find('div', {'class': "col-9"}) bae = soup.find('div', {'class': "col-9"})
@ -57,11 +53,6 @@ def tor2door_description_parser(soup):
# half_star = bae[2].find('i', {'class': "fas fa-star-half-alt"}) # half_star = bae[2].find('i', {'class': "fas fa-star-half-alt"})
# rating = len(full_stars) + (0.5 if half_star is not None else 0) # rating = len(full_stars) + (0.5 if half_star is not None else 0)
# Finding Warranty
escrow = mb[2].text
escrow = escrow.replace("Payment:", "")
escrow = escrow.strip()
# Finding Quantity Sold and Left # Finding Quantity Sold and Left
temp = mb[4].text.split(',') temp = mb[4].text.split(',')
@ -116,8 +107,8 @@ def tor2door_description_parser(soup):
MS = MS.replace('\n', '') MS = MS.replace('\n', '')
# Populating the final variable (this should be a list with all fields scraped) # Populating the final variable (this should be a list with all fields scraped)
row = (name, describe, lastSeen, rules, CVE, MS, review, category, shipFrom, shipTo, left, escrow, terms, vendor, row = (vendor, rating_vendor, success, name, describe, CVE, MS, category, views, reviews, rating_item, addDate,
sold, addDate, available, endDate, BTC, USD, rating, success, EURO) BTC, USD, EURO, sold, left, shipFrom, shipTo)
# Sending the results # Sending the results
return row return row
@ -128,17 +119,19 @@ def tor2door_listing_parser(soup):
# Fields to be parsed # Fields to be parsed
nm = 0 # Total_Products (Should be Integer) nm = 0 # Total_Products (Should be Integer)
mktName = "Tor2door" # 0 Marketplace_Name mktName = "Tor2door" # 0 Marketplace_Name
vendor = [] # 18 Vendor y
rating_vendor = [] # 19 Vendor_Rating
success = [] # 20 Vendor_Successful_Transactions
name = [] # 1 Product_Name y name = [] # 1 Product_Name y
CVE = [] # 2 Product_CVE_Classification (Common Vulnerabilities and Exposures) CVE = [] # 2 Product_CVE_Classification (Common Vulnerabilities and Exposures)
MS = [] # 3 Product_MS_Classification (Microsoft Security) MS = [] # 3 Product_MS_Classification (Microsoft Security)
category = [] # 4 Product_Category y category = [] # 4 Product_Category y
describe = [] # 5 Product_Description describe = [] # 5 Product_Description
escrow = [] # 6 Vendor_Warranty
views = [] # 7 Product_Number_Of_Views views = [] # 7 Product_Number_Of_Views
reviews = [] # 8 Product_Number_Of_Reviews y reviews = [] # 7 Product_Number_Of_Reviews
rating_item = [] # 8 Product_Rating
addDate = [] # 9 Product_AddDate addDate = [] # 9 Product_AddDate
lastSeen = [] # 10 Product_LastViewDate
BTC = [] # 11 Product_BTC_SellingPrice BTC = [] # 11 Product_BTC_SellingPrice
USD = [] # 12 Product_USD_SellingPrice y USD = [] # 12 Product_USD_SellingPrice y
EURO = [] # 13 Product_EURO_SellingPrice EURO = [] # 13 Product_EURO_SellingPrice
@ -146,10 +139,7 @@ def tor2door_listing_parser(soup):
qLeft =[] # 15 Product_QuantityLeft qLeft =[] # 15 Product_QuantityLeft
shipFrom = [] # 16 Product_ShippedFrom shipFrom = [] # 16 Product_ShippedFrom
shipTo = [] # 17 Product_ShippedTo shipTo = [] # 17 Product_ShippedTo
vendor = [] # 18 Vendor y href = [] # 24 Product_Links
rating = [] # 19 Vendor_Rating
success = [] # 20 Vendor_Successful_Transactions
href = [] # 24 Product_Links (Urls)
listing = soup.findAll('div', {"class": "card product-card mb-3"}) listing = soup.findAll('div', {"class": "card product-card mb-3"})
@ -228,8 +218,8 @@ def tor2door_listing_parser(soup):
MS.append(MSValue) MS.append(MSValue)
# Populate the final variable (this should be a list with all fields scraped) # Populate the final variable (this should be a list with all fields scraped)
return organizeProducts(mktName, nm, name, CVE, MS, category, describe, escrow, views, reviews, addDate, lastSeen, return organizeProducts(mktName, nm, vendor, rating_vendor, success, name, CVE, MS, category, describe, views,
BTC, USD, EURO, qLeft, shipFrom, shipTo, vendor, rating, success, sold, href) reviews, rating_item, addDate, BTC, USD, EURO, sold, qLeft, shipFrom, shipTo, href)
def tor2door_links_parser(soup): def tor2door_links_parser(soup):


+ 28
- 26
MarketPlaces/Utilities/utilities.py View File

@ -207,57 +207,59 @@ def cleanLink(originalLink):
return originalLink return originalLink
def organizeProducts(marketplace, nm, nombre, CVE, MS, category, describe, escrow, views, reviews, addDate, lastSeen, def organizeProducts(marketplace, nm, vendor, rating_vendor, success_vendor, nombre, CVE, MS, category, describe,
BTC, USD, EURO, qLeft, shipFrom, shipTo, user, rating, success, sold, href): views, reviews, rating_item, addDate, BTC, USD, EURO, sold, qLeft, shipFrom, shipTo, href):
rw = [] rw = []
day = time.strftime("%m/%d/%Y") day = time.strftime("%m/%d/%Y")
#day = time.strftime("%d/%m/%Y")
ahora = time.strftime("%I:%M:%S") ahora = time.strftime("%I:%M:%S")
for n in range(nm): for n in range(nm):
lne = marketplace + "," #0 lne = marketplace # 0
lne += "-1" if len(nombre) == 0 else nombre[n] #1 lne += ","
lne += vendor[n] # 1
lne += ","
lne += "-1" if len(rating_vendor) == 0 else rating_vendor[n] # 2
lne += ","
lne += "-1" if len(success_vendor) == 0 else success_vendor[n] # 3
lne += ","
lne += nombre[n] # 4
lne += ',' lne += ','
lne += "-1" if len(CVE) == 0 else CVE[n] #2 lne += "-1" if len(describe) == 0 else describe[n] # 5
lne += "," lne += ","
lne += "-1" if len(MS) == 0 else MS[n] #3 lne += "-1" if len(CVE) == 0 else CVE[n] # 6
lne += "," lne += ","
lne += "-1" if len(category) == 0 else category[n] #4 lne += "-1" if len(MS) == 0 else MS[n] # 7
lne += "," lne += ","
lne += "-1" if len(describe) == 0 else describe[n] #5 lne += "-1" if len(category) == 0 else category[n] # 8
lne += "," lne += ","
lne += "-1" if len(escrow) == 0 else escrow[n] #6 lne += "-1" if len(views) == 0 else views[n] # 9
lne += "," lne += ","
lne += "-1" if len(views) == 0 else views[n] #7 lne += "-1" if len(reviews) == 0 else reviews[n] # 10
lne += "," lne += ","
lne += "-1" if len(reviews) == 0 else reviews[n] #8 lne += "-1" if len(rating_item) == 0 else rating_item[n] # 11
lne += "," lne += ","
lne += "-1" if len(addDate) == 0 else addDate[n] #9 lne += "-1" if len(addDate) == 0 else addDate[n] # 12
lne += "," lne += ","
lne += "-1" if len(lastSeen) == 0 else lastSeen[n] #10 lne += "-1" if len(BTC) == 0 else BTC[n] # 13
lne += "," lne += ","
lne += "-1" if len(BTC) == 0 else BTC[n] #11 lne += "-1" if len(USD) == 0 else USD[n] # 14
lne += "," lne += ","
lne += "-1" if len(USD) == 0 else USD[n] #12 lne += "-1" if len(EURO) == 0 else EURO[n] # 15
lne += "," lne += ","
lne += "-1" if len(EURO) == 0 else EURO[n] #13 lne += "-1" if len(sold) == 0 else sold[n] # 16
lne += "," lne += ","
lne += "-1" if len(sold) == 0 else sold[n] #14 lne += "-1" if len(qLeft) == 0 else qLeft[n] # 17
lne += "," lne += ","
lne += "-1" if len(qLeft) == 0 else qLeft[n] #15 lne += "-1" if len(shipFrom) == 0 else shipFrom[n] # 18
lne += "," lne += ","
lne += "-1" if len(shipFrom) == 0 else shipFrom[n] #16 lne += "-1" if len(shipTo) == 0 else shipTo[n] # 19
lne += "," lne += ","
lne += "-1" if len(shipTo) == 0 else shipTo[n] #17 lne += "-1" if len(href) == 0 else href[n] # 20
lne += "," + user[n] + "," #18
lne += "-1" if len(rating) == 0 else rating[n] #19
lne += "," lne += ","
lne += "-1" if len(success) == 0 else success[n] #20 lne += day + " " + ahora # 21
lne += "," + "-1" + "," + day + " " + ahora + "," #21, 22
lne += "-1" if len(href) == 0 else href[n] #23
rw.append(lne) rw.append(lne)


Write Preview
|||||||
​x
 
​
000:0
Loading…
Cancel
Save