python - NLTK perplexity error -
i got error message
traceback (most recent call last): file "c:/users/shengrong/desktop/bigram", line 55, in <module> bg = bigram(file) file "c:/users/shengrong/desktop/bigram", line 43, in bigram return tt1.perplexity(my_bigrams) file "c:\python27\lib\site-packages\nltk\model\ngram.py", line 217, in perplexity return pow(2.0, self.entropy(text)) file "c:\python27\lib\site-packages\nltk\model\ngram.py", line 205, in entropy e += self.logprob(token, context) file "c:\python27\lib\site-packages\nltk\model\ngram.py", line 151, in logprob return -log(self.prob(word, context), 2) valueerror: math domain error import os,csv,nltk nltk.model.ngram import ngrammodel nltk.probability import lidstoneprobdist fout = open("/users/shengrong/documents/personal/wn1.data.csv", "w") outfilehandle = csv.writer(fout, delimiter=",", quotechar='"', quoting=csv.quote_nonnumeric) localrow = [] localrow.append("file name") localrow.append("perplexity unigram") localrow.append("perplexity bigram") localrow.append("perplexity trigram") outfilehandle.writerow(localrow) def bigram(file): file_object = open(file) ln=file_object.read() words = nltk.word_tokenize(ln) my_bigrams = nltk.bigrams(words) my_trigrams = nltk.trigrams(words) tt1=ngrammodel(2, my_bigrams, estimator = none) return tt1.perplexity(my_bigrams) #set path of folder os.chdir("/users/shengrong/documents/a") s = os.getcwd() #search files in folder files = os.listdir(s) file in files: bg = bigram(file) localrow= [] localrow.append(file) localrow.append(bg) outfilehandle.writerow(localrow) fout.close() how fix problem? without using loop read folder, code working well.
thank guys.
the error getting math domain error. since works without loop, may 1 or more of files contains data cannot processed math package.
please confirm files in folder contains data in correct format , expected values.
Comments
Post a Comment