You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
392 lines
12 KiB
392 lines
12 KiB
__author__ = 'DarkWeb'
|
|
|
|
import psycopg2
|
|
import traceback
|
|
import time
|
|
from datetime import date
|
|
|
|
'''
|
|
user
|
|
add
|
|
forum_id
|
|
user_reputation
|
|
status_user
|
|
interest_user
|
|
|
|
post
|
|
remove
|
|
user_reputation
|
|
status_user
|
|
interest_user
|
|
'''
|
|
|
|
def connectDataBase():
|
|
|
|
try:
|
|
|
|
return psycopg2.connect(host='localhost', user='postgres', password='password', dbname='darkweb_markets_forums')
|
|
|
|
except:
|
|
|
|
print ("Data base (darkweb_forums) not found.")
|
|
raise SystemExit
|
|
|
|
|
|
def verifyForum(cur, nameForum):
|
|
|
|
try:
|
|
|
|
cur.execute("select forum_id from forums where name_forum = %(nameForum)s limit 1", {'nameForum': nameForum})
|
|
|
|
recset = cur.fetchall()
|
|
|
|
if recset:
|
|
return recset[0][0]
|
|
else:
|
|
return 0
|
|
|
|
except:
|
|
|
|
trace = traceback.format_exc()
|
|
print (trace)
|
|
|
|
|
|
def verifyBoard(cur, forum, nameBoard):
|
|
|
|
try:
|
|
|
|
cur.execute("select board_id from boards where forum_id = %(forum)s and name_board = %(nameBoard)s limit 1",
|
|
{'forum': forum, 'nameBoard': nameBoard})
|
|
|
|
recset = cur.fetchall()
|
|
|
|
if recset:
|
|
return recset[0][0]
|
|
else:
|
|
return 0
|
|
|
|
except:
|
|
|
|
trace = traceback.format_exc()
|
|
print (trace)
|
|
|
|
|
|
def verifyTopic(cur, forumId, boardId, authorId, titleTopic):
|
|
|
|
try:
|
|
|
|
cur.execute("select topic_id from topics where forum_id = %(forumId)s and board_id = %(boardId)s and "
|
|
"author_id = %(authorId)s and title_topic = %(titleTopic)s limit 1", {'forumId': forumId,
|
|
'boardId': boardId,
|
|
'authorId': authorId,
|
|
'titleTopic': titleTopic})
|
|
|
|
recset = cur.fetchall()
|
|
|
|
if recset:
|
|
return recset[0][0]
|
|
else:
|
|
return 0
|
|
|
|
except:
|
|
|
|
trace = traceback.format_exc()
|
|
print (trace)
|
|
|
|
|
|
def verifyUser(cur, nameUser, forumId):
|
|
|
|
try:
|
|
|
|
cur.execute("select user_id from users where name_user = %(nameUser)s and forum_id = %(forumId)s limit 1",
|
|
{'nameUser': nameUser, 'forumId': forumId})
|
|
|
|
recset = cur.fetchall()
|
|
|
|
if recset:
|
|
return recset[0][0]
|
|
else:
|
|
return 0
|
|
|
|
except:
|
|
|
|
trace = traceback.format_exc()
|
|
print (trace)
|
|
|
|
|
|
def getLastForum(cur):
|
|
|
|
try:
|
|
|
|
cur.execute("select forum_id from forums order by forum_id desc limit 1")
|
|
|
|
recset = cur.fetchall()
|
|
|
|
if recset:
|
|
return recset[0][0]
|
|
else:
|
|
return 0
|
|
|
|
except:
|
|
|
|
trace = traceback.format_exc()
|
|
print (trace)
|
|
|
|
|
|
def getLastBoard(cur):
|
|
|
|
try:
|
|
|
|
cur.execute("select board_id from boards order by board_id desc limit 1")
|
|
|
|
recset = cur.fetchall()
|
|
|
|
if recset:
|
|
return recset[0][0]
|
|
else:
|
|
return 0
|
|
|
|
except:
|
|
|
|
trace = traceback.format_exc()
|
|
print (trace)
|
|
|
|
|
|
def getLastTopic(cur):
|
|
|
|
try:
|
|
|
|
cur.execute("select topic_id from topics order by topic_id desc limit 1")
|
|
|
|
recset = cur.fetchall()
|
|
|
|
if recset:
|
|
return recset[0][0]
|
|
else:
|
|
return 0
|
|
|
|
except:
|
|
|
|
trace = traceback.format_exc()
|
|
print (trace)
|
|
|
|
|
|
def getLastUser(cur):
|
|
|
|
try:
|
|
|
|
cur.execute("select user_id from users order by user_id desc")
|
|
|
|
recset = cur.fetchall()
|
|
|
|
if recset:
|
|
return recset[0][0]
|
|
else:
|
|
return 0
|
|
|
|
except:
|
|
|
|
trace = traceback.format_exc()
|
|
print (trace)
|
|
|
|
|
|
'''
|
|
def getLastPost(cur):
|
|
|
|
try:
|
|
|
|
cur.execute("select id from Posts order by id desc limit 1")
|
|
|
|
recset = cur.fetchall()
|
|
|
|
if recset:
|
|
return recset[0][0]
|
|
else:
|
|
return 0
|
|
|
|
except:
|
|
|
|
trace = traceback.format_exc()
|
|
print (trace)
|
|
'''
|
|
|
|
|
|
def create_forum(cur, row):
|
|
|
|
forumId = verifyForum(cur, row[0])
|
|
|
|
if not forumId:
|
|
|
|
forumId = int(getLastForum(cur) + 1)
|
|
|
|
sql = "Insert into forums (forum_id, name_forum, url_forum, dateinserted_forum) Values (%s, %s, %s, %s)"
|
|
|
|
recset = [forumId, row[0], None, row[8]]
|
|
|
|
cur.execute(sql, recset)
|
|
|
|
return forumId
|
|
|
|
|
|
def create_board(cur, row, forumId):
|
|
|
|
boardId = verifyBoard(cur, forumId, row[1])
|
|
|
|
if not boardId:
|
|
|
|
boardId = int(getLastBoard(cur) + 1)
|
|
|
|
sql = "Insert into boards (board_id, forum_id, name_board, dateinserted_board) Values (%s, %s, %s, %s)"
|
|
|
|
recset = [boardId, forumId, row[1], row[8]]
|
|
|
|
cur.execute(sql, recset)
|
|
|
|
return boardId
|
|
|
|
|
|
def create_topic(cur, row, forumId, boardId, authorId):
|
|
|
|
topicId = verifyTopic(cur, forumId, boardId, authorId, row[3])
|
|
|
|
if not topicId:
|
|
|
|
topicId = int(getLastTopic(cur) + 1)
|
|
|
|
sql = "Insert into topics (topic_id, forum_id, board_id, author_id, title_topic, views_topic, posts_topic, " \
|
|
"href_topic, dateadded_topic, dateinserted_topic, classification_topic) Values (%s, %s, %s, %s, %s, %s, " \
|
|
"%s, %s, %s, %s, %s)"
|
|
|
|
recset = [topicId, forumId, boardId, authorId,
|
|
row[3],
|
|
row[4] if row[4] != '-1' else None,
|
|
row[5] if row[5] != '-1' else None,
|
|
row[6] if row[6] != '-1' else None,
|
|
row[7] if row[7] != '-1' else None,
|
|
row[8],
|
|
row[17]]
|
|
cur.execute(sql, recset)
|
|
|
|
return topicId
|
|
|
|
|
|
def create_user(cur, row, forumId, index):
|
|
|
|
userId = verifyUser(cur, row[9][index], forumId)
|
|
|
|
if not userId:
|
|
|
|
userId = int(getLastUser(cur) + 1)
|
|
|
|
sql = "Insert into users (user_id, forum_id, name_user, status_user, reputation_user, interest_user, " \
|
|
"signature_user, dateinserted_user) Values (%s, %s, %s, %s, %s, %s, %s, %s)"
|
|
|
|
recset = [userId, forumId,
|
|
row[9][index],
|
|
row[10][index] if row[10][index] != '-1' else None,
|
|
row[11][index] if row[11][index] != '-1' else None,
|
|
row[12][index] if row[12][index] != '-1' else None,
|
|
row[13][index] if row[13][index] != '-1' else None,
|
|
row[8]]
|
|
|
|
cur.execute(sql, recset)
|
|
|
|
return userId
|
|
|
|
|
|
def create_posts(cur, row, forumId, boardId, topicId):
|
|
|
|
if row[9] != "-1":
|
|
|
|
for i in range(len(row[9])):
|
|
|
|
if i != 0:
|
|
userId = create_user(cur, row, forumId, i)
|
|
else:
|
|
userId = verifyUser(cur, row[2], forumId)
|
|
|
|
sql = "Insert into posts (forum_id, board_id, topic_id, user_id, content_post, feedback_post, " \
|
|
"dateadded_post, dateinserted_post) Values (%s, %s, %s, %s, %s, %s, %s, %s)"
|
|
|
|
recset = [forumId, boardId, topicId, userId,
|
|
row[14][i] if row[14][i] != '-1' else None,
|
|
row[15][i] if row[15][i] != '-1' else None,
|
|
row[16][i] if row[16][i] != '-1' else None,
|
|
row[8]]
|
|
|
|
cur.execute(sql, recset)
|
|
|
|
|
|
def create_database(cur, con):
|
|
|
|
try:
|
|
|
|
sql = "create table forums (forum_id integer NOT NULL, name_forum character varying(255) NOT NULL, url_forum " \
|
|
"character varying(255) null, dateinserted_forum timestamp(6) with time zone NOT NULL, constraint " \
|
|
"forums_pk primary key (forum_id))"
|
|
cur.execute(sql)
|
|
|
|
sql = "create table boards (board_id integer NOT NULL, forum_id integer NOT NULL, name_board character " \
|
|
"varying(255) NOT NULL, dateinserted_board timestamp(6) with time zone NOT NULL, constraint boards_pk " \
|
|
"primary key (board_id), constraint boards_forum_id_fkey foreign key (forum_id) references forums (" \
|
|
"forum_id))"
|
|
cur.execute(sql)
|
|
|
|
sql = "create table users (user_id integer NOT NULL, forum_id integer NOT NULL, name_user character varying(" \
|
|
"255) NOT NULL, status_user character varying(255) null, reputation_user character varying(255) null, " \
|
|
"interest_user character varying(5000) null, signature_user character varying(1000) null, " \
|
|
"dateinserted_user timestamp(6) with time zone NOT NULL, constraint users_pk primary key (user_id), " \
|
|
"constraint users_forum_id_fkey foreign key (forum_id) references forums (forum_id))"
|
|
cur.execute(sql)
|
|
|
|
sql = "create table users_history(user_id integer NOT NULL, forum_id integer NOT NULL, name_user character " \
|
|
"varying(255) NOT NULL, status_user character varying(255) null, reputation_user character varying(255) " \
|
|
"null, interest_user character varying(5000) null, signature_user character varying(1000) null, " \
|
|
"dateinserted_user timestamp(6) with time zone NOT NULL, constraint users_history_pk primary key (" \
|
|
"user_id, dateinserted_user), constraint users_history_user_id_fkey foreign key (user_id) references " \
|
|
"users (user_id), constraint users_history_forum_id_fkey foreign key (forum_id) references forums (" \
|
|
"forum_id))"
|
|
cur.execute(sql)
|
|
|
|
sql = "create table topics(topic_id integer NOT NULL, forum_id integer NOT NULL, board_id integer NOT NULL, " \
|
|
"author_id integer NOT NULL, title_topic character varying(255) NOT NULL, views_topic integer null, " \
|
|
"posts_topic integer null, href_topic character varying(255) null, dateadded_topic timestamp(6) with " \
|
|
"time zone null, dateinserted_topic timestamp(6) with time zone NOT NULL, classification_topic double " \
|
|
"precision NOT NULL, constraint topics_pk primary key (topic_id), constraint topics_author_id_fkey " \
|
|
"foreign key (author_id) references users (user_id), constraint topics_board_id_fkey foreign key (" \
|
|
"board_id) references boards (board_id), constraint topics_forum_id_fkey foreign key (forum_id) " \
|
|
"references forums (forum_id))"
|
|
cur.execute(sql)
|
|
|
|
sql = "create table topics_history(topic_id integer NOT NULL, forum_id integer NOT NULL, board_id integer NOT " \
|
|
"NULL, author_id integer NOT NULL, title_topic character varying(255) NOT NULL, views_topic integer " \
|
|
"null, posts_topic integer null, href_topic character varying(255) null, dateadded_topic timestamp(6) " \
|
|
"with time zone null, dateinserted_topic timestamp(6) with time zone NOT NULL, classification_topic " \
|
|
"double precision NOT NULL, constraint topics_history_pk primary key (topic_id, dateinserted_topic), " \
|
|
"constraint topics_history_topic_id_fkey foreign key (topic_id) references topics (topic_id), " \
|
|
"constraint topics_history_author_id_fkey foreign key (author_id) references users (user_id), " \
|
|
"constraint topics_history_board_id_fkey foreign key (board_id) references boards (board_id), " \
|
|
"constraint topics_history_forum_id_fkey foreign key (forum_id) references forums (forum_id))"
|
|
cur.execute(sql)
|
|
|
|
sql = "create table posts(forum_id integer NOT NULL, board_id integer NOT NULL, topic_id integer NOT NULL, " \
|
|
"user_id integer NOT NULL, content_post character varying(100000) null, feedback_post integer null, " \
|
|
"dateadded_post timestamp(6) with time zone NOT NULL, dateinserted_post timestamp(6) with time zone NOT " \
|
|
"NULL, constraint posts_pk primary key (forum_id, board_id, topic_id, user_id, dateadded_post), " \
|
|
"constraint posts_author_id_fkey foreign key (user_id) references users (user_id), constraint " \
|
|
"posts_board_id_fkey foreign key (board_id) references boards (board_id), constraint " \
|
|
"posts_forum_id_fkey foreign key (forum_id) references forums (forum_id), constraint " \
|
|
"posts_topic_id_fkey foreign key (topic_id) references topics (topic_id))"
|
|
cur.execute(sql)
|
|
|
|
con.commit()
|
|
|
|
except:
|
|
|
|
con.rollback()
|
|
|
|
trace = traceback.format_exc()
|
|
|
|
if (trace.find("already exists")==-1):
|
|
print ("There was a problem during the database creation." )
|
|
raise SystemExit
|