From 9fca859758f2621e6051cb85ed751427a23c6c67 Mon Sep 17 00:00:00 2001
From: ericssonmarin-cpp <85146518+ericssonmarin-cpp@users.noreply.github.com>
Date: Sun, 15 Oct 2023 14:00:28 -0700
Subject: [PATCH] Forums and Markets status.
---
.idea/DW_Pipeline_Test.iml | 2 +-
.idea/misc.xml | 2 +-
Forums/DB_Connection/db_connection.py | 23 +++++++++++++++++++
Forums/Initialization/forumsList.txt | 9 +-------
Forums/Initialization/prepare_parser.py | 6 +++++
MarketPlaces/DB_Connection/db_connection.py | 23 +++++++++++++++++++
MarketPlaces/Initialization/marketsList.txt | 2 +-
MarketPlaces/Initialization/prepare_parser.py | 6 +++++
setup.ini | 14 ++++++-----
9 files changed, 70 insertions(+), 17 deletions(-)
diff --git a/.idea/DW_Pipeline_Test.iml b/.idea/DW_Pipeline_Test.iml
index 8489f64..9f9af70 100644
--- a/.idea/DW_Pipeline_Test.iml
+++ b/.idea/DW_Pipeline_Test.iml
@@ -2,7 +2,7 @@
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
- <orderEntry type="jdk" jdkName="C:\Users\calsyslab\anaconda3" jdkType="Python SDK" />
+ <orderEntry type="jdk" jdkName="C:\Users\santanamarin\Anaconda3" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>
\ No newline at end of file
diff --git a/.idea/misc.xml b/.idea/misc.xml
index 11f1ea0..653c6ff 100644
--- a/.idea/misc.xml
+++ b/.idea/misc.xml
@@ -1,4 +1,4 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
- <component name="ProjectRootManager" version="2" project-jdk-name="C:\Users\calsyslab\anaconda3" project-jdk-type="Python SDK" />
+ <component name="ProjectRootManager" version="2" project-jdk-name="C:\Users\santanamarin\Anaconda3" project-jdk-type="Python SDK" />
</project>
\ No newline at end of file
diff --git a/Forums/DB_Connection/db_connection.py b/Forums/DB_Connection/db_connection.py
index e4f6c5d..f0d4ed6 100644
--- a/Forums/DB_Connection/db_connection.py
+++ b/Forums/DB_Connection/db_connection.py
@@ -484,6 +484,24 @@ def create_posts(cur, row, forumId, topicId):
'dateinserted_post': row[8],
'postId': postId})
+def create_status(cur, forumId, date, status):
+
+ date = datetime.strptime(date, "%m%d%Y")
+
+ # checking if status already exists
+ sql = "select * from forums_status where forum_id = %(forum_id)s and date_inserted = %(date_inserted)s"
+ cur.execute(sql, {'forum_id': forumId, 'date_inserted': date})
+
+ recset = cur.fetchall()
+ if recset:
+ sql = "Update forums_status set status = %(status)s where forum_id = %(forum_id)s and date_inserted = %(date_inserted)s"
+ recset = {'status': status, 'forum_id': forumId, 'date_inserted': date}
+ else:
+ sql = "Insert into forums_status (forum_id, date_inserted, status) Values (%s, %s, %s)"
+ recset = [forumId, date, status]
+
+ cur.execute(sql, recset)
+
def create_database(cur, con):
try:
@@ -496,6 +514,11 @@ def create_database(cur, con):
sql = "create unique index unique_forum ON forums USING btree (name_forum ASC NULLS LAST)"
cur.execute(sql)
+ sql = "Create table forums_status (forum_id integer NOT NULL, date_inserted date NOT NULL, status bit(1) NOT NULL, " \
+ "CONSTRAINT forums_log_pkey PRIMARY KEY (forum_id, date_inserted), " \
+ "CONSTRAINT forums_fk FOREIGN KEY (forum_id) REFERENCES forums (forum_id))"
+ cur.execute(sql)
+
sql = "create table users (user_id integer NOT NULL, forum_id integer NOT NULL, name_user character varying(" \
"255) NOT NULL, status_user character varying(255) null, reputation_user character varying(255) null, " \
"interest_user character varying(5000) null, signature_user character varying(1000) null, " \
diff --git a/Forums/Initialization/forumsList.txt b/Forums/Initialization/forumsList.txt
index 9cfeb56..efa9686 100644
--- a/Forums/Initialization/forumsList.txt
+++ b/Forums/Initialization/forumsList.txt
@@ -1,8 +1 @@
-Altenens
-BestCardingWorld
-Cardingleaks
-CryptBB
-HiddenAnswers
-Libre
-OnniForums
-Procrax
\ No newline at end of file
+BestCardingWorld
\ No newline at end of file
diff --git a/Forums/Initialization/prepare_parser.py b/Forums/Initialization/prepare_parser.py
index 91b662f..1f55319 100644
--- a/Forums/Initialization/prepare_parser.py
+++ b/Forums/Initialization/prepare_parser.py
@@ -341,6 +341,12 @@ def new_parse(forum, url, createLog):
# move listing files of completed folder
move_file(listingFile, createLog, logFile)
+ # registering the current forum status (up/down) in the database
+ forumId = verifyForum(cur, forum)
+ if (forumId > 0):
+ create_status(cur, forumId, CURRENT_DATE, '1' if len(listings) > 0 else '0')
+ con.commit()
+
if createLog:
logFile.close()
diff --git a/MarketPlaces/DB_Connection/db_connection.py b/MarketPlaces/DB_Connection/db_connection.py
index 8769869..4f439f0 100644
--- a/MarketPlaces/DB_Connection/db_connection.py
+++ b/MarketPlaces/DB_Connection/db_connection.py
@@ -401,6 +401,24 @@ def create_items(cur, row, marketId, vendorId):
return itemId
+def create_status(cur, marketId, date, status):
+
+ date = datetime.strptime(date, "%m%d%Y")
+
+ # checking if status already exists
+ sql = "select * from marketplaces_status where market_id = %(market_id)s and date_inserted = %(date_inserted)s"
+ cur.execute(sql, {'market_id': marketId, 'date_inserted': date})
+
+ recset = cur.fetchall()
+ if recset:
+ sql = "Update marketplaces_status set status = %(status)s where market_id = %(market_id)s and date_inserted = %(date_inserted)s"
+ recset = {'status': status, 'market_id': marketId, 'date_inserted': date}
+ else:
+ sql = "Insert into marketplaces_status (market_id, date_inserted, status) Values (%s, %s, %s)"
+ recset = [marketId, date, status]
+
+ cur.execute(sql, recset)
+
def create_database(cur, con):
try:
@@ -413,6 +431,11 @@ def create_database(cur, con):
sql = "create unique index unique_market ON marketplaces USING btree (name_market ASC NULLS LAST)"
cur.execute(sql)
+ sql = "Create table marketplaces_status (market_id integer NOT NULL, date_inserted date NOT NULL, status bit(1) NOT NULL, " \
+ "CONSTRAINT marketplaces_log_pkey PRIMARY KEY (market_id, date_inserted), " \
+ "CONSTRAINT marketplaces_fk FOREIGN KEY (market_id) REFERENCES marketplaces (market_id))"
+ cur.execute(sql)
+
sql = "create table vendors(vendor_id integer not null, market_id integer not null, name_vendor character " \
"varying(255) not null, rating_vendor character varying(255), successfultransactions_vendor integer " \
"null, image_vendor character varying(10000000) null, dateinserted_vendor timestamp(6) with time zone not null, " \
diff --git a/MarketPlaces/Initialization/marketsList.txt b/MarketPlaces/Initialization/marketsList.txt
index 87f811c..8b944c5 100644
--- a/MarketPlaces/Initialization/marketsList.txt
+++ b/MarketPlaces/Initialization/marketsList.txt
@@ -1 +1 @@
-ThiefWorld
\ No newline at end of file
+ViceCity
\ No newline at end of file
diff --git a/MarketPlaces/Initialization/prepare_parser.py b/MarketPlaces/Initialization/prepare_parser.py
index e075541..c56054e 100644
--- a/MarketPlaces/Initialization/prepare_parser.py
+++ b/MarketPlaces/Initialization/prepare_parser.py
@@ -363,6 +363,12 @@ def new_parse(marketPlace, url, createLog):
# move listing files of completed folder
move_file(listingFile, createLog, logFile)
+ # registering the current forum status (up/down) in the database
+ marketId = verifyMarketPlace(cur, marketPlace)
+ if (marketId > 0):
+ create_status(cur, marketId, CURRENT_DATE, '1' if len(listings) > 0 else '0')
+ con.commit()
+
if createLog:
logFile.close()
diff --git a/setup.ini b/setup.ini
index 29997a6..883e495 100644
--- a/setup.ini
+++ b/setup.ini
@@ -1,17 +1,19 @@
[TOR]
-firefox_binary_path = C:\Users\calsyslab\Desktop\Tor Browser\Browser\firefox.exe
-firefox_profile_path = C:\Users\calsyslab\Desktop\Tor Browser\Browser\TorBrowser\Data\Browser\profile.default
-geckodriver_path = C:\calsyslab\Project\dw_pipeline_test\selenium\geckodriver.exe
+firefox_binary_path = C:\Users\santanamarin\OneDrive - Cal Poly Pomona\Desktop\Tor Browser\Browser\firefox.exe
+firefox_profile_path = C:\Users\santanamarin\OneDrive - Cal Poly Pomona\Desktop\Tor Browser\Browser\TorBrowser\Data\Browser\profile.default
+geckodriver_path = E:\Faculty\CSPUP\ResearchProjects\dw_pipeline_test\selenium\geckodriver.exe
+
+"C:\Users\santanamarin\OneDrive - Cal Poly Pomona\Desktop\Tor Browser\Browser\firefox.exe"
[Project]
-project_directory = C:\calsyslab\Project\dw_pipeline_test
-shared_folder = \\VBoxSvr\Shared
+project_directory = E:\Faculty\CSPUP\ResearchProjects\dw_pipeline_test
+shared_folder = E:\Faculty\CSPUP\ResearchProjects\dw_pipeline_test
[PostgreSQL]
ip = localhost
username = postgres
-password = password
+password = 123
database = darkweb_markets_forums
[Encryption]