Revert "Forums and Markets status."

This reverts commit 9fca859758

.idea/DW_Pipeline_Test.iml

@@ -2,7 +2,7 @@
 <module type="PYTHON_MODULE" version="4">
   <component name="NewModuleRootManager">
     <content url="file://$MODULE_DIR$" />
-    <orderEntry type="jdk" jdkName="C:\Users\santanamarin\Anaconda3" jdkType="Python SDK" />
+    <orderEntry type="jdk" jdkName="C:\Users\calsyslab\anaconda3" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
 </module>

.idea/misc.xml

@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
-  <component name="ProjectRootManager" version="2" project-jdk-name="C:\Users\santanamarin\Anaconda3" project-jdk-type="Python SDK" />
+  <component name="ProjectRootManager" version="2" project-jdk-name="C:\Users\calsyslab\anaconda3" project-jdk-type="Python SDK" />
 </project>

Forums/DB_Connection/db_connection.py

@@ -484,24 +484,6 @@ def create_posts(cur, row, forumId, topicId):
                                       'dateinserted_post': row[8],
                                       'postId': postId})
 
-def create_status(cur, forumId, date, status):
-
-    date = datetime.strptime(date, "%m%d%Y")
-
-    # checking if status already exists
-    sql = "select * from forums_status where forum_id = %(forum_id)s and date_inserted = %(date_inserted)s"
-    cur.execute(sql, {'forum_id': forumId, 'date_inserted': date})
-
-    recset = cur.fetchall()
-    if recset:
-       sql = "Update forums_status set status = %(status)s where forum_id = %(forum_id)s and date_inserted = %(date_inserted)s"
-       recset = {'status': status, 'forum_id': forumId, 'date_inserted': date}
-    else:
-       sql = "Insert into forums_status (forum_id, date_inserted, status) Values (%s, %s, %s)"
-       recset = [forumId, date, status]
-
-    cur.execute(sql, recset)
-
 def create_database(cur, con):
 
     try:
@@ -514,11 +496,6 @@ def create_database(cur, con):
         sql = "create unique index unique_forum ON forums USING btree (name_forum ASC NULLS LAST)"
         cur.execute(sql)
 
-        sql = "Create table forums_status (forum_id integer NOT NULL, date_inserted date NOT NULL, status bit(1) NOT NULL, " \
-              "CONSTRAINT forums_log_pkey PRIMARY KEY (forum_id, date_inserted), " \
-              "CONSTRAINT forums_fk FOREIGN KEY (forum_id) REFERENCES forums (forum_id))"
-        cur.execute(sql)
-
         sql = "create table users (user_id integer NOT NULL, forum_id integer NOT NULL, name_user character varying(" \
               "255) NOT NULL, status_user character varying(255) null, reputation_user character varying(255) null, " \
               "interest_user character varying(5000) null, signature_user character varying(1000) null, " \

Forums/Initialization/forumsList.txt

@@ -1 +1,8 @@
-BestCardingWorld
+Altenens
+BestCardingWorld
+Cardingleaks
+CryptBB
+HiddenAnswers
+Libre
+OnniForums
+Procrax

Forums/Initialization/prepare_parser.py

@@ -341,12 +341,6 @@ def new_parse(forum, url, createLog):
                 # move listing files of completed folder
                 move_file(listingFile, createLog, logFile)
 
-    # registering the current forum status (up/down) in the database
-    forumId = verifyForum(cur, forum)
-    if (forumId > 0):
-        create_status(cur, forumId, CURRENT_DATE, '1' if len(listings) > 0 else '0')
-        con.commit()
-
     if createLog:
         logFile.close()
 

MarketPlaces/DB_Connection/db_connection.py

@@ -401,24 +401,6 @@ def create_items(cur, row, marketId, vendorId):
 
     return itemId
 
-def create_status(cur, marketId, date, status):
-
-    date = datetime.strptime(date, "%m%d%Y")
-
-    # checking if status already exists
-    sql = "select * from marketplaces_status where market_id = %(market_id)s and date_inserted = %(date_inserted)s"
-    cur.execute(sql, {'market_id': marketId, 'date_inserted': date})
-
-    recset = cur.fetchall()
-    if recset:
-       sql = "Update marketplaces_status set status = %(status)s where market_id = %(market_id)s and date_inserted = %(date_inserted)s"
-       recset = {'status': status, 'market_id': marketId, 'date_inserted': date}
-    else:
-       sql = "Insert into marketplaces_status (market_id, date_inserted, status) Values (%s, %s, %s)"
-       recset = [marketId, date, status]
-
-    cur.execute(sql, recset)
-
 def create_database(cur, con):
 
     try:
@@ -431,11 +413,6 @@ def create_database(cur, con):
         sql = "create unique index unique_market ON marketplaces USING btree (name_market ASC NULLS LAST)"
         cur.execute(sql)
 
-        sql = "Create table marketplaces_status (market_id integer NOT NULL, date_inserted date NOT NULL, status bit(1) NOT NULL, " \
-              "CONSTRAINT marketplaces_log_pkey PRIMARY KEY (market_id, date_inserted), " \
-              "CONSTRAINT marketplaces_fk FOREIGN KEY (market_id) REFERENCES marketplaces (market_id))"
-        cur.execute(sql)
-
         sql = "create table vendors(vendor_id integer not null, market_id integer not null, name_vendor character " \
               "varying(255) not null, rating_vendor character varying(255), successfultransactions_vendor integer " \
               "null, image_vendor character varying(10000000) null, dateinserted_vendor timestamp(6) with time zone not null, " \

MarketPlaces/Initialization/marketsList.txt

@@ -1 +1 @@
-ViceCity
+ThiefWorld

MarketPlaces/Initialization/prepare_parser.py

@@ -363,12 +363,6 @@ def new_parse(marketPlace, url, createLog):
                 # move listing files of completed folder
                 move_file(listingFile, createLog, logFile)
 
-    # registering the current forum status (up/down) in the database
-    marketId = verifyMarketPlace(cur, marketPlace)
-    if (marketId > 0):
-        create_status(cur, marketId, CURRENT_DATE, '1' if len(listings) > 0 else '0')
-        con.commit()
-
     if createLog:
         logFile.close()
 

setup.ini

@@ -1,19 +1,17 @@
 
 [TOR]
-firefox_binary_path = C:\Users\santanamarin\OneDrive - Cal Poly Pomona\Desktop\Tor Browser\Browser\firefox.exe
-firefox_profile_path = C:\Users\santanamarin\OneDrive - Cal Poly Pomona\Desktop\Tor Browser\Browser\TorBrowser\Data\Browser\profile.default
-geckodriver_path = E:\Faculty\CSPUP\ResearchProjects\dw_pipeline_test\selenium\geckodriver.exe
-
-"C:\Users\santanamarin\OneDrive - Cal Poly Pomona\Desktop\Tor Browser\Browser\firefox.exe"
+firefox_binary_path = C:\Users\calsyslab\Desktop\Tor Browser\Browser\firefox.exe
+firefox_profile_path = C:\Users\calsyslab\Desktop\Tor Browser\Browser\TorBrowser\Data\Browser\profile.default
+geckodriver_path = C:\calsyslab\Project\dw_pipeline_test\selenium\geckodriver.exe
 
 [Project]
-project_directory = E:\Faculty\CSPUP\ResearchProjects\dw_pipeline_test
-shared_folder = E:\Faculty\CSPUP\ResearchProjects\dw_pipeline_test
+project_directory = C:\calsyslab\Project\dw_pipeline_test
+shared_folder = \\VBoxSvr\Shared
 
 [PostgreSQL]
 ip = localhost
 username = postgres
-password = 123
+password = password
 database = darkweb_markets_forums
 
 [Encryption]