diff --git a/.idea/DW_Pipeline_Test.iml b/.idea/DW_Pipeline_Test.iml index 9f9af70..8489f64 100644 --- a/.idea/DW_Pipeline_Test.iml +++ b/.idea/DW_Pipeline_Test.iml @@ -2,7 +2,7 @@ - + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml index 653c6ff..11f1ea0 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -1,4 +1,4 @@ - + \ No newline at end of file diff --git a/Forums/DB_Connection/db_connection.py b/Forums/DB_Connection/db_connection.py index f0d4ed6..e4f6c5d 100644 --- a/Forums/DB_Connection/db_connection.py +++ b/Forums/DB_Connection/db_connection.py @@ -484,24 +484,6 @@ def create_posts(cur, row, forumId, topicId): 'dateinserted_post': row[8], 'postId': postId}) -def create_status(cur, forumId, date, status): - - date = datetime.strptime(date, "%m%d%Y") - - # checking if status already exists - sql = "select * from forums_status where forum_id = %(forum_id)s and date_inserted = %(date_inserted)s" - cur.execute(sql, {'forum_id': forumId, 'date_inserted': date}) - - recset = cur.fetchall() - if recset: - sql = "Update forums_status set status = %(status)s where forum_id = %(forum_id)s and date_inserted = %(date_inserted)s" - recset = {'status': status, 'forum_id': forumId, 'date_inserted': date} - else: - sql = "Insert into forums_status (forum_id, date_inserted, status) Values (%s, %s, %s)" - recset = [forumId, date, status] - - cur.execute(sql, recset) - def create_database(cur, con): try: @@ -514,11 +496,6 @@ def create_database(cur, con): sql = "create unique index unique_forum ON forums USING btree (name_forum ASC NULLS LAST)" cur.execute(sql) - sql = "Create table forums_status (forum_id integer NOT NULL, date_inserted date NOT NULL, status bit(1) NOT NULL, " \ - "CONSTRAINT forums_log_pkey PRIMARY KEY (forum_id, date_inserted), " \ - "CONSTRAINT forums_fk FOREIGN KEY (forum_id) REFERENCES forums (forum_id))" - cur.execute(sql) - sql = "create table users (user_id integer NOT NULL, forum_id integer NOT NULL, name_user character varying(" \ "255) NOT NULL, status_user character varying(255) null, reputation_user character varying(255) null, " \ "interest_user character varying(5000) null, signature_user character varying(1000) null, " \ diff --git a/Forums/Initialization/forumsList.txt b/Forums/Initialization/forumsList.txt index efa9686..9cfeb56 100644 --- a/Forums/Initialization/forumsList.txt +++ b/Forums/Initialization/forumsList.txt @@ -1 +1,8 @@ -BestCardingWorld \ No newline at end of file +Altenens +BestCardingWorld +Cardingleaks +CryptBB +HiddenAnswers +Libre +OnniForums +Procrax \ No newline at end of file diff --git a/Forums/Initialization/prepare_parser.py b/Forums/Initialization/prepare_parser.py index 1f55319..91b662f 100644 --- a/Forums/Initialization/prepare_parser.py +++ b/Forums/Initialization/prepare_parser.py @@ -341,12 +341,6 @@ def new_parse(forum, url, createLog): # move listing files of completed folder move_file(listingFile, createLog, logFile) - # registering the current forum status (up/down) in the database - forumId = verifyForum(cur, forum) - if (forumId > 0): - create_status(cur, forumId, CURRENT_DATE, '1' if len(listings) > 0 else '0') - con.commit() - if createLog: logFile.close() diff --git a/MarketPlaces/DB_Connection/db_connection.py b/MarketPlaces/DB_Connection/db_connection.py index 4f439f0..8769869 100644 --- a/MarketPlaces/DB_Connection/db_connection.py +++ b/MarketPlaces/DB_Connection/db_connection.py @@ -401,24 +401,6 @@ def create_items(cur, row, marketId, vendorId): return itemId -def create_status(cur, marketId, date, status): - - date = datetime.strptime(date, "%m%d%Y") - - # checking if status already exists - sql = "select * from marketplaces_status where market_id = %(market_id)s and date_inserted = %(date_inserted)s" - cur.execute(sql, {'market_id': marketId, 'date_inserted': date}) - - recset = cur.fetchall() - if recset: - sql = "Update marketplaces_status set status = %(status)s where market_id = %(market_id)s and date_inserted = %(date_inserted)s" - recset = {'status': status, 'market_id': marketId, 'date_inserted': date} - else: - sql = "Insert into marketplaces_status (market_id, date_inserted, status) Values (%s, %s, %s)" - recset = [marketId, date, status] - - cur.execute(sql, recset) - def create_database(cur, con): try: @@ -431,11 +413,6 @@ def create_database(cur, con): sql = "create unique index unique_market ON marketplaces USING btree (name_market ASC NULLS LAST)" cur.execute(sql) - sql = "Create table marketplaces_status (market_id integer NOT NULL, date_inserted date NOT NULL, status bit(1) NOT NULL, " \ - "CONSTRAINT marketplaces_log_pkey PRIMARY KEY (market_id, date_inserted), " \ - "CONSTRAINT marketplaces_fk FOREIGN KEY (market_id) REFERENCES marketplaces (market_id))" - cur.execute(sql) - sql = "create table vendors(vendor_id integer not null, market_id integer not null, name_vendor character " \ "varying(255) not null, rating_vendor character varying(255), successfultransactions_vendor integer " \ "null, image_vendor character varying(10000000) null, dateinserted_vendor timestamp(6) with time zone not null, " \ diff --git a/MarketPlaces/Initialization/marketsList.txt b/MarketPlaces/Initialization/marketsList.txt index 8b944c5..87f811c 100644 --- a/MarketPlaces/Initialization/marketsList.txt +++ b/MarketPlaces/Initialization/marketsList.txt @@ -1 +1 @@ -ViceCity \ No newline at end of file +ThiefWorld \ No newline at end of file diff --git a/MarketPlaces/Initialization/prepare_parser.py b/MarketPlaces/Initialization/prepare_parser.py index c56054e..e075541 100644 --- a/MarketPlaces/Initialization/prepare_parser.py +++ b/MarketPlaces/Initialization/prepare_parser.py @@ -363,12 +363,6 @@ def new_parse(marketPlace, url, createLog): # move listing files of completed folder move_file(listingFile, createLog, logFile) - # registering the current forum status (up/down) in the database - marketId = verifyMarketPlace(cur, marketPlace) - if (marketId > 0): - create_status(cur, marketId, CURRENT_DATE, '1' if len(listings) > 0 else '0') - con.commit() - if createLog: logFile.close() diff --git a/setup.ini b/setup.ini index 883e495..29997a6 100644 --- a/setup.ini +++ b/setup.ini @@ -1,19 +1,17 @@ [TOR] -firefox_binary_path = C:\Users\santanamarin\OneDrive - Cal Poly Pomona\Desktop\Tor Browser\Browser\firefox.exe -firefox_profile_path = C:\Users\santanamarin\OneDrive - Cal Poly Pomona\Desktop\Tor Browser\Browser\TorBrowser\Data\Browser\profile.default -geckodriver_path = E:\Faculty\CSPUP\ResearchProjects\dw_pipeline_test\selenium\geckodriver.exe - -"C:\Users\santanamarin\OneDrive - Cal Poly Pomona\Desktop\Tor Browser\Browser\firefox.exe" +firefox_binary_path = C:\Users\calsyslab\Desktop\Tor Browser\Browser\firefox.exe +firefox_profile_path = C:\Users\calsyslab\Desktop\Tor Browser\Browser\TorBrowser\Data\Browser\profile.default +geckodriver_path = C:\calsyslab\Project\dw_pipeline_test\selenium\geckodriver.exe [Project] -project_directory = E:\Faculty\CSPUP\ResearchProjects\dw_pipeline_test -shared_folder = E:\Faculty\CSPUP\ResearchProjects\dw_pipeline_test +project_directory = C:\calsyslab\Project\dw_pipeline_test +shared_folder = \\VBoxSvr\Shared [PostgreSQL] ip = localhost username = postgres -password = 123 +password = password database = darkweb_markets_forums [Encryption]