.

project.py/Project_STBI.py

+#!/usr/bin/env python
+# coding: utf-8
+
+# In[1]:
+
+
+import string
+import re
+from sklearn.feature_extraction.text import CountVectorizer
+import xml.dom.minidom as minidom
+dcmnt_xml = minidom.parse("Dataset_Article.xml")
+
+
+# In[2]:
+
+
+all_doc_no = dcmnt_xml.getElementsByTagName('Id')
+all_profile = dcmnt_xml.getElementsByTagName('title')
+all_date = dcmnt_xml.getElementsByTagName('year')
+all_text = dcmnt_xml.getElementsByTagName('content')
+all_pub = dcmnt_xml.getElementsByTagName('author')
+
+
+N_DOC_sample = len(all_doc_no)
+
+
+# In[3]:
+
+
+print(N_DOC_sample)
+
+
+# In[4]:
+
+
+all_sentence_doc_sample = []
+for i in range(N_DOC_sample):
+    sentence_doc_sample = ' '+ all_text[i].firstChild.data
+    all_sentence_doc_sample.append(sentence_doc_sample)
+
+
+# In[5]:
+
+
+all_sentence_doc_sample
+
+
+# ## Preprocessing 
+
+# In[6]:
+
+
+tokens_doc = []
+
+
+# In[7]:
+
+
+def remove_punc_tokenize(sentence):
+    tokens = []
+    for punctuation in string.punctuation:
+        sentence = sentence.replace(punctuation," ")
+    
+    sentence = re.sub(r'^https?:\/\/.*[\r\n]*', '', sentence, flags=re.MULTILINE)
+    for w in CountVectorizer().build_tokenizer()(sentence):
+        tokens.append(w)
+    return tokens
+
+
+# In[8]:
+
+
+for i in range(N_DOC_sample):
+    tokens_doc.append(remove_punc_tokenize(all_sentence_doc_sample[i]))
+
+
+# In[9]:
+
+
+tokens_doc
+
+
+# In[10]:
+
+
+from nltk.corpus import stopwords
+stop_words = set(stopwords.words('english'))
+def stop_word_token(tokens):
+    tokens = [w for w in tokens if not w in stop_words]
+    return tokens
+
+for i in range(N_DOC_sample):
+    tokens_doc[i] = stop_word_token(tokens_doc[i])
+
+
+# In[11]:
+
+
+for i in range(N_DOC_sample):
+    tokens_doc[i] = ([w for w in tokens_doc[i] if not any(j.isdigit() for j in w)])
+
+
+# In[12]:
+
+
+tokens_doc
+
+
+# In[13]:
+
+
+from nltk.stem import PorterStemmer
+stemmer = PorterStemmer()
+def stemming(tokens):
+    for i in range(0, len(tokens)):
+        if (tokens[i] != stemmer.stem(tokens[i])):
+            tokens[i] = stemmer.stem(tokens[i])
+    return tokens
+
+
+for i in range(N_DOC_sample):
+    tokens_doc[i] = stemming(tokens_doc[i])
+
+
+# In[14]:
+
+
+all_tokens = []
+for i in range(N_DOC_sample):
+    for w in tokens_doc[i]:
+        all_tokens.append(w)
+
+new_sentence = ' '.join([w for w in all_tokens])
+
+for w in CountVectorizer().build_tokenizer()(new_sentence):
+    all_tokens.append(w)
+
+
+# In[15]:
+
+
+all_tokens
+
+
+# In[16]:
+
+
+from itertools import count
+try: 
+    from itertools import izip as zip
+except ImportError:
+    pass
+proximity_index = {}
+for token in all_tokens:
+    dict_doc_position = {}
+    for n in range(N_DOC_sample):
+        if(token in tokens_doc[n]):
+            dict_doc_position[all_doc_no[n].firstChild.data] = [i+1 for i, j in zip(count(), tokens_doc[n]) if j == token]
+    proximity_index[token] = dict_doc_position
+
+
+# In[17]:
+
+
+import collections
+proximity_index = collections.OrderedDict(sorted(proximity_index.items()))
+for key, value in proximity_index.items():
+    print (key, value)
+