|
|
- import codecs, re, time
-
- def is_unicode(s):
- try:
- str(s)
- return False
- except:
- return True
-
- def contains_num(s):
- nums = range(10)
- str_nums = [str(num) for num in nums]
- char_set = set(s)
- for num in str_nums:
- if num in char_set:
- return True
- return False
-
-
- from nltk.tokenize import regexp_tokenize, wordpunct_tokenize
- from nltk.corpus import stopwords
- from nltk.stem.snowball import SnowballStemmer
- import string, re
- stemmer = SnowballStemmer('english')
-
- word_matcher = re.compile(u'[^\W\d_]+', re.UNICODE)
- def is_unicode(s):
- if word_matcher.match(s):
- return True
- return False
-
- def stem_preprocessor(s):
- return my_preprocessor(s, stem=True)
-
- def my_preprocessor(s, stem=False):
- pattern = u'[^\W\d_]+|[^\w\s]+|\d+'
- tokens = regexp_tokenize(s, pattern)
- cleaned_tokens = []
- for token in tokens:
- if token and is_unicode(token) and not token in stopwords.words('english'):
- cleaned_token = stemmer.stem(token.lower()) if stem==True else token.lower()
- cleaned_tokens.append(cleaned_token)
-
- return ' '.join(cleaned_tokens)
-
-
- regex = re.compile(u'[%s]' % re.escape(string.punctuation)) #see documentation here: http://docs.python.org/2/library/string.html
- def my_preprocessor2(s):
- tokens = wordpunct_tokenize(s)
- cleaned_tokens = [regex.sub(u'', token) for token in tokens]
- return u' '.join(cleaned_tokens)
-
- from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
- from sklearn.linear_model import SGDClassifier
- from sklearn.pipeline import make_pipeline
- from sklearn.cross_validation import train_test_split
- from sklearn.metrics import classification_report
- from sklearn.neighbors import NearestCentroid
- from sklearn.cross_validation import KFold
-
- def KFold_classification_report(clf, docs, labels, K=10):
- y_pred = [-1] * len(docs)
- cv = KFold(len(docs), K, shuffle=True)
- for traincv, testcv in cv:
- train_docs = [docs[i] for i in traincv]
- train_labels = [labels[i] for i in traincv]
-
- clf.fit(train_docs, train_labels)
-
- for i in testcv:
- y_pred[i] = clf.predict([docs[i]])[0]
-
- return classification_report(labels, y_pred)
-
|
