__author__ = 'DarkWeb' import psycopg2 import traceback import configparser def connectDataBase(): try: config = configparser.ConfigParser() config.read('../../setup.ini') ip = config.get('PostgreSQL', 'ip') username = config.get('PostgreSQL', 'username') password = config.get('PostgreSQL', 'password') database = config.get('PostgreSQL', 'database') return psycopg2.connect(host=ip, user=username, password=password, dbname=database) except: print ("Data base (darkweb_forums) not found.") raise SystemExit def verifyForum(cur, nameForum): try: cur.execute("select forum_id from forums where name_forum = %(nameForum)s limit 1", {'nameForum': nameForum}) recset = cur.fetchall() if recset: return recset[0][0] else: return 0 except: trace = traceback.format_exc() print (trace) def verifyBoard(cur, forum, nameBoard): try: cur.execute("select board_id from boards where forum_id = %(forum)s and name_board = %(nameBoard)s limit 1", {'forum': forum, 'nameBoard': nameBoard}) recset = cur.fetchall() if recset: return recset[0][0] else: return 0 except: trace = traceback.format_exc() print (trace) def verifyTopic(cur, forumId, boardId, authorId, titleTopic): try: cur.execute("select topic_id from topics where forum_id = %(forumId)s and board_id = %(boardId)s and " "author_id = %(authorId)s and title_topic = %(titleTopic)s limit 1", {'forumId': forumId, 'boardId': boardId, 'authorId': authorId, 'titleTopic': titleTopic}) recset = cur.fetchall() if recset: return recset[0][0] else: return 0 except: trace = traceback.format_exc() print (trace) def verifyUser(cur, nameUser, forumId): try: cur.execute("select user_id from users where name_user = %(nameUser)s and forum_id = %(forumId)s limit 1", {'nameUser': nameUser, 'forumId': forumId}) recset = cur.fetchall() if recset: return recset[0][0] else: return 0 except: trace = traceback.format_exc() print (trace) def getLastForum(cur): try: cur.execute("select forum_id from forums order by forum_id desc limit 1") recset = cur.fetchall() if recset: return recset[0][0] else: return 0 except: trace = traceback.format_exc() print (trace) def getLastBoard(cur): try: cur.execute("select board_id from boards order by board_id desc limit 1") recset = cur.fetchall() if recset: return recset[0][0] else: return 0 except: trace = traceback.format_exc() print (trace) def getLastTopic(cur): try: cur.execute("select topic_id from topics order by topic_id desc limit 1") recset = cur.fetchall() if recset: return recset[0][0] else: return 0 except: trace = traceback.format_exc() print (trace) def getLastUser(cur): try: cur.execute("select user_id from users order by user_id desc") recset = cur.fetchall() if recset: return recset[0][0] else: return 0 except: trace = traceback.format_exc() print (trace) ''' def getLastPost(cur): try: cur.execute("select id from Posts order by id desc limit 1") recset = cur.fetchall() if recset: return recset[0][0] else: return 0 except: trace = traceback.format_exc() print (trace) ''' def create_forum(cur, row, url): forumId = verifyForum(cur, row[0]) if not forumId: forumId = int(getLastForum(cur) + 1) sql = "Insert into forums (forum_id, name_forum, url_forum, dateinserted_forum) Values (%s, %s, %s, %s)" recset = [forumId, row[0], url, row[8]] cur.execute(sql, recset) return forumId def create_board(cur, row, forumId): boardId = verifyBoard(cur, forumId, row[1]) if not boardId: boardId = int(getLastBoard(cur) + 1) sql = "Insert into boards (board_id, forum_id, name_board, dateinserted_board) Values (%s, %s, %s, %s)" recset = [boardId, forumId, row[1], row[8]] cur.execute(sql, recset) return boardId def create_topic(cur, row, forumId, boardId, authorId): topicId = verifyTopic(cur, forumId, boardId, authorId, row[3]) if not topicId: topicId = int(getLastTopic(cur) + 1) sql = "Insert into topics (topic_id, forum_id, board_id, author_id, title_topic, views_topic, posts_topic, " \ "href_topic, dateadded_topic, dateinserted_topic, classification_topic) Values (%s, %s, %s, %s, %s, %s, " \ "%s, %s, %s, %s, %s)" recset = [topicId, forumId, boardId, authorId, row[3], row[4] if row[4] != '-1' else None, row[5] if row[5] != '-1' else None, row[6] if row[6] != '-1' else None, row[7] if row[7] != '-1' else None, row[8], row[17]] cur.execute(sql, recset) return topicId def create_user(cur, row, forumId, index): userId = verifyUser(cur, row[9][index], forumId) if not userId: userId = int(getLastUser(cur) + 1) sql = "Insert into users (user_id, forum_id, name_user, status_user, reputation_user, interest_user, " \ "signature_user, dateinserted_user) Values (%s, %s, %s, %s, %s, %s, %s, %s)" recset = [userId, forumId, row[9][index], row[10][index] if row[10][index] != '-1' else None, row[11][index] if row[11][index] != '-1' else None, row[12][index] if row[12][index] != '-1' else None, row[13][index] if row[13][index] != '-1' else None, row[8]] cur.execute(sql, recset) return userId def create_posts(cur, row, forumId, boardId, topicId): if row[9] != "-1": for i in range(len(row[9])): if i != 0: userId = create_user(cur, row, forumId, i) else: userId = verifyUser(cur, row[2], forumId) sql = "Insert into posts (forum_id, board_id, topic_id, user_id, content_post, feedback_post, " \ "dateadded_post, dateinserted_post) Values (%s, %s, %s, %s, %s, %s, %s, %s)" recset = [forumId, boardId, topicId, userId, row[14][i] if row[14][i] != '-1' else None, row[15][i] if row[15][i] != '-1' else None, row[16][i] if row[16][i] != '-1' else None, row[8]] cur.execute(sql, recset) def create_database(cur, con): try: sql = "create table forums (forum_id integer NOT NULL, name_forum character varying(255) NOT NULL, url_forum " \ "character varying(255) null, dateinserted_forum timestamp(6) with time zone NOT NULL, constraint " \ "forums_pk primary key (forum_id))" cur.execute(sql) sql = "create table boards (board_id integer NOT NULL, forum_id integer NOT NULL, name_board character " \ "varying(255) NOT NULL, dateinserted_board timestamp(6) with time zone NOT NULL, constraint boards_pk " \ "primary key (board_id), constraint boards_forum_id_fkey foreign key (forum_id) references forums (" \ "forum_id))" cur.execute(sql) sql = "create table users (user_id integer NOT NULL, forum_id integer NOT NULL, name_user character varying(" \ "255) NOT NULL, status_user character varying(255) null, reputation_user character varying(255) null, " \ "interest_user character varying(5000) null, signature_user character varying(1000) null, " \ "dateinserted_user timestamp(6) with time zone NOT NULL, constraint users_pk primary key (user_id), " \ "constraint users_forum_id_fkey foreign key (forum_id) references forums (forum_id))" cur.execute(sql) sql = "create table users_history(user_id integer NOT NULL, forum_id integer NOT NULL, name_user character " \ "varying(255) NOT NULL, status_user character varying(255) null, reputation_user character varying(255) " \ "null, interest_user character varying(5000) null, signature_user character varying(1000) null, " \ "dateinserted_user timestamp(6) with time zone NOT NULL, constraint users_history_pk primary key (" \ "user_id, dateinserted_user), constraint users_history_user_id_fkey foreign key (user_id) references " \ "users (user_id), constraint users_history_forum_id_fkey foreign key (forum_id) references forums (" \ "forum_id))" cur.execute(sql) sql = "create table topics(topic_id integer NOT NULL, forum_id integer NOT NULL, board_id integer NOT NULL, " \ "author_id integer NOT NULL, title_topic character varying(255) NOT NULL, views_topic integer null, " \ "posts_topic integer null, href_topic character varying(255) null, dateadded_topic timestamp(6) with " \ "time zone null, dateinserted_topic timestamp(6) with time zone NOT NULL, classification_topic double " \ "precision NOT NULL, constraint topics_pk primary key (topic_id), constraint topics_author_id_fkey " \ "foreign key (author_id) references users (user_id), constraint topics_board_id_fkey foreign key (" \ "board_id) references boards (board_id), constraint topics_forum_id_fkey foreign key (forum_id) " \ "references forums (forum_id))" cur.execute(sql) sql = "create table topics_history(topic_id integer NOT NULL, forum_id integer NOT NULL, board_id integer NOT " \ "NULL, author_id integer NOT NULL, title_topic character varying(255) NOT NULL, views_topic integer " \ "null, posts_topic integer null, href_topic character varying(255) null, dateadded_topic timestamp(6) " \ "with time zone null, dateinserted_topic timestamp(6) with time zone NOT NULL, classification_topic " \ "double precision NOT NULL, constraint topics_history_pk primary key (topic_id, dateinserted_topic), " \ "constraint topics_history_topic_id_fkey foreign key (topic_id) references topics (topic_id), " \ "constraint topics_history_author_id_fkey foreign key (author_id) references users (user_id), " \ "constraint topics_history_board_id_fkey foreign key (board_id) references boards (board_id), " \ "constraint topics_history_forum_id_fkey foreign key (forum_id) references forums (forum_id))" cur.execute(sql) sql = "create table posts(forum_id integer NOT NULL, board_id integer NOT NULL, topic_id integer NOT NULL, " \ "user_id integer NOT NULL, content_post character varying(100000) null, feedback_post integer null, " \ "dateadded_post timestamp(6) with time zone NOT NULL, dateinserted_post timestamp(6) with time zone NOT " \ "NULL, constraint posts_pk primary key (forum_id, board_id, topic_id, user_id, dateadded_post), " \ "constraint posts_author_id_fkey foreign key (user_id) references users (user_id), constraint " \ "posts_board_id_fkey foreign key (board_id) references boards (board_id), constraint " \ "posts_forum_id_fkey foreign key (forum_id) references forums (forum_id), constraint " \ "posts_topic_id_fkey foreign key (topic_id) references topics (topic_id))" cur.execute(sql) con.commit() except: con.rollback() trace = traceback.format_exc() if (trace.find("already exists")==-1): print ("There was a problem during the database creation." ) raise SystemExit