From b084d76d3ed6277bbb6e030828be564503321d0c Mon Sep 17 00:00:00 2001 From: westernmeadow Date: Fri, 27 Oct 2023 15:13:58 -0700 Subject: [PATCH] listings, descriptions, and reference date --- Forums/DB_Connection/db_connection.py | 19 +++++++++------ Forums/Initialization/prepare_parser.py | 8 +++++-- MarketPlaces/DB_Connection/db_connection.py | 24 ++++++++++++------- MarketPlaces/Initialization/prepare_parser.py | 8 +++++-- 4 files changed, 39 insertions(+), 20 deletions(-) diff --git a/Forums/DB_Connection/db_connection.py b/Forums/DB_Connection/db_connection.py index f0d4ed6..dfdec49 100644 --- a/Forums/DB_Connection/db_connection.py +++ b/Forums/DB_Connection/db_connection.py @@ -3,7 +3,7 @@ __author__ = 'DarkWeb' import psycopg2 import traceback from Forums.Utilities.utilities import * - +from dateutil.relativedelta import relativedelta, FR def connectDataBase(): @@ -484,21 +484,25 @@ def create_posts(cur, row, forumId, topicId): 'dateinserted_post': row[8], 'postId': postId}) -def create_status(cur, forumId, date, status): +def create_status(cur, forumId, date, listings, descriptions, status): date = datetime.strptime(date, "%m%d%Y") + # getting last Fridays a reference date + date_reference = date + relativedelta(weekday=FR(-1)) + # checking if status already exists sql = "select * from forums_status where forum_id = %(forum_id)s and date_inserted = %(date_inserted)s" cur.execute(sql, {'forum_id': forumId, 'date_inserted': date}) recset = cur.fetchall() if recset: - sql = "Update forums_status set status = %(status)s where forum_id = %(forum_id)s and date_inserted = %(date_inserted)s" - recset = {'status': status, 'forum_id': forumId, 'date_inserted': date} + sql = "Update forums_status set listings = %(listings)s, descriptions = %(descriptions)s, status = %(status)s, date_reference = %(date_reference)s " \ + "where forum_id = %(forum_id)s and date_inserted = %(date_inserted)s" + recset = {'listings': listings, 'descriptions': descriptions, 'status': status, 'date_reference': date_reference, 'forum_id': forumId, 'date_inserted': date} else: - sql = "Insert into forums_status (forum_id, date_inserted, status) Values (%s, %s, %s)" - recset = [forumId, date, status] + sql = "Insert into forums_status (forum_id, date_inserted, listings, descriptions, status, date_reference) Values (%s, %s, %s, %s, %s, %s)" + recset = [forumId, date, listings, descriptions, status, date_reference] cur.execute(sql, recset) @@ -514,7 +518,8 @@ def create_database(cur, con): sql = "create unique index unique_forum ON forums USING btree (name_forum ASC NULLS LAST)" cur.execute(sql) - sql = "Create table forums_status (forum_id integer NOT NULL, date_inserted date NOT NULL, status bit(1) NOT NULL, " \ + sql = "Create table forums_status (forum_id integer NOT NULL, date_inserted date NOT NULL, " \ + "listings integer NOT NULL, descriptions integer NOT NULL, status bit(1) NOT NULL, date_reference date NOT NULL " \ "CONSTRAINT forums_log_pkey PRIMARY KEY (forum_id, date_inserted), " \ "CONSTRAINT forums_fk FOREIGN KEY (forum_id) REFERENCES forums (forum_id))" cur.execute(sql) diff --git a/Forums/Initialization/prepare_parser.py b/Forums/Initialization/prepare_parser.py index 1f55319..31982fd 100644 --- a/Forums/Initialization/prepare_parser.py +++ b/Forums/Initialization/prepare_parser.py @@ -341,10 +341,14 @@ def new_parse(forum, url, createLog): # move listing files of completed folder move_file(listingFile, createLog, logFile) - # registering the current forum status (up/down) in the database + # registering the current forum status (up/down) and the number of scraped pages in the database forumId = verifyForum(cur, forum) if (forumId > 0): - create_status(cur, forumId, CURRENT_DATE, '1' if len(listings) > 0 else '0') + + readListings = glob.glob(os.path.join(mainDir, CURRENT_DATE + "\\Listing\\read", '*.htm')) + readDescriptions = glob.glob(os.path.join(mainDir, CURRENT_DATE + "\\Description\\read", '*.htm')) + + create_status(cur, forumId, CURRENT_DATE, len(readListings), len(readDescriptions), '1' if len(listings) > 0 else '0') con.commit() if createLog: diff --git a/MarketPlaces/DB_Connection/db_connection.py b/MarketPlaces/DB_Connection/db_connection.py index 4f439f0..2f3341a 100644 --- a/MarketPlaces/DB_Connection/db_connection.py +++ b/MarketPlaces/DB_Connection/db_connection.py @@ -4,7 +4,7 @@ import psycopg2 import traceback import configparser from MarketPlaces.Utilities.utilities import * - +from dateutil.relativedelta import relativedelta, FR def connectDataBase(): @@ -273,6 +273,8 @@ def create_items(cur, row, marketId, vendorId): if newItem: + # decode_decrypt_image_in_base64(row[20]) + sql = "Insert into items (item_id, market_id, vendor_id, name_item, description_item, cve_item, ms_item, category_item, " \ "views_item, reviews_item, rating_item, dateadded_item, btc_item, usd_item, euro_item, quantitysold_item, " \ "quantityleft_item, shippedfrom_item, shippedto_item, lastseen_item, image_item, href_item, dateinserted_item, " \ @@ -312,7 +314,7 @@ def create_items(cur, row, marketId, vendorId): recset = cur.fetchall() - # decode_decrypt_image_in_base64(recset[0][20]) + # decode_decrypt_image_in_base64(recset[0]['image_item']) if (str(recset[0]['description_item']) != str(row[5] if row[5] != '-1' else None) or str(recset[0]['cve_item']) != str(row[6] if row[6] != '-1' else None) or @@ -401,24 +403,27 @@ def create_items(cur, row, marketId, vendorId): return itemId -def create_status(cur, marketId, date, status): +def create_status(cur, marketId, date, listings, descriptions, status): date = datetime.strptime(date, "%m%d%Y") + # getting last Fridays a reference date + date_reference = date + relativedelta(weekday=FR(-1)) + # checking if status already exists sql = "select * from marketplaces_status where market_id = %(market_id)s and date_inserted = %(date_inserted)s" cur.execute(sql, {'market_id': marketId, 'date_inserted': date}) recset = cur.fetchall() if recset: - sql = "Update marketplaces_status set status = %(status)s where market_id = %(market_id)s and date_inserted = %(date_inserted)s" - recset = {'status': status, 'market_id': marketId, 'date_inserted': date} + sql = "Update marketplaces_status set listings = %(listings)s, descriptions = %(descriptions)s, status = %(status)s, date_reference = %(date_reference)s " \ + "where market_id = %(market_id)s and date_inserted = %(date_inserted)s" + recset = {'listings': listings, 'descriptions': descriptions, 'status': status, 'date_reference': date_reference, 'market_id': marketId, 'date_inserted': date} else: - sql = "Insert into marketplaces_status (market_id, date_inserted, status) Values (%s, %s, %s)" - recset = [marketId, date, status] + sql = "Insert into marketplaces_status (market_id, date_inserted, listings, descriptions, status, date_reference) Values (%s, %s, %s, %s, %s, %s)" + recset = [marketId, date, listings, descriptions, status, date_reference] cur.execute(sql, recset) - def create_database(cur, con): try: @@ -431,7 +436,8 @@ def create_database(cur, con): sql = "create unique index unique_market ON marketplaces USING btree (name_market ASC NULLS LAST)" cur.execute(sql) - sql = "Create table marketplaces_status (market_id integer NOT NULL, date_inserted date NOT NULL, status bit(1) NOT NULL, " \ + sql = "Create table marketplaces_status (market_id integer NOT NULL, date_inserted date NOT NULL, " \ + "listings integer NOT NULL, descriptions integer NOT NULL, status bit(1) NOT NULL, date_reference date NOT NULL " \ "CONSTRAINT marketplaces_log_pkey PRIMARY KEY (market_id, date_inserted), " \ "CONSTRAINT marketplaces_fk FOREIGN KEY (market_id) REFERENCES marketplaces (market_id))" cur.execute(sql) diff --git a/MarketPlaces/Initialization/prepare_parser.py b/MarketPlaces/Initialization/prepare_parser.py index 982995f..7c35f5a 100644 --- a/MarketPlaces/Initialization/prepare_parser.py +++ b/MarketPlaces/Initialization/prepare_parser.py @@ -369,10 +369,14 @@ def new_parse(marketPlace, url, createLog): # move listing files of completed folder move_file(listingFile, createLog, logFile) - # registering the current market status (up/down) in the database + # registering the current forum status (up/down) and the number of scraped pages in the database marketId = verifyMarketPlace(cur, marketPlace) if (marketId > 0): - create_status(cur, marketId, CURRENT_DATE, '1' if len(listings) > 0 else '0') + + readListings = glob.glob(os.path.join(mainDir, CURRENT_DATE + "\\Listing\\read", '*.htm')) + readDescriptions = glob.glob(os.path.join(mainDir, CURRENT_DATE + "\\Description\\read", '*.htm')) + + create_status(cur, marketId, CURRENT_DATE, len(readListings), len(readDescriptions), '1' if len(listings) > 0 else '0') con.commit() if createLog: