diff --git a/.gitignore b/.gitignore index e967e92..251cbdd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ .DS_Store -takahe/takahe.pyc +*.pyc +__pycache__ +*.dot diff --git a/README.md b/README.md index 2e8e985..58ccaa5 100644 --- a/README.md +++ b/README.md @@ -11,28 +11,29 @@ A keyphrase-based reranking method can be applied to generate more informative c ## Dependancies -As of today, takahe is built for Python 2. +As of today, takahe is built for Python 3.7. You may need to install the following libraries : -- [networkx](http://networkx.github.io/) (installation guide is available [here](http://networkx.github.io/documentation/latest/install.html)) +- [networkx=1.11](http://networkx.github.io/) (installation guide is available [here](http://networkx.github.io/documentation/latest/install.html)) - [graphviz](http://www.graphviz.org/) and graphviz-dev - [pygraphviz](http://pygraphviz.github.io/documentation/latest/install.html) +- [pydotplus](https://pypi.org/project/pydotplus/) ## Example A typical usage of this module is: - + import takahe - + # Create a word graph from the set of sentences with parameters : # - minimal number of words in the compression : 6 # - language of the input sentences : en (english) # - POS tag for punctuation marks : PUNCT - compresser = takahe.word_graph( sentences, - nb_words = 6, - lang = 'en', + compresser = takahe.word_graph( sentences, + nb_words = 6, + lang = 'en', punct_tag = "PUNCT" ) # Get the 50 best paths @@ -52,13 +53,13 @@ A typical usage of this module is: # 2. Rerank compressions by keyphrases (Boudin and Morin's method) reranker = takahe.keyphrase_reranker( sentences, - candidates, + candidates, lang = 'en' ) reranked_candidates = reranker.rerank_nbest_compressions() # Loop over the best reranked candidates for score, path in reranked_candidates: - + # Print the best reranked candidates print round(score, 3), ' '.join([u[0] for u in path]) diff --git a/example.py b/example.py index a6bb798..fbc90f4 100644 --- a/example.py +++ b/example.py @@ -1,7 +1,7 @@ #!/usr/bin/python # -*- coding: utf-8 -*- -import takahe +from takahe import takahe ################################################################################ @@ -10,7 +10,7 @@ Monday/NNP ./PUNCT", "Hillary/NNP Clinton/NNP wanted/VBD to/TO visit/VB China/NNP \ last/JJ month/NN but/CC postponed/VBD her/PRP$ plans/NNS till/IN Monday/NNP \ last/JJ week/NN ./PUNCT", "Hillary/NNP Clinton/NNP paid/VBD a/DT visit/NN to/TO \ -the/DT People/NNP Republic/NNP of/IN China/NNP on/IN Monday/NNP ./PUNCT", +the/DT People/NNP Republic/NNP of/IN China/NNP on/IN Monday/NNP ./PUNCT", "Last/JJ week/NN the/DT Secretary/NNP of/IN State/NNP Ms./NNP Clinton/NNP \ visited/VBD Chinese/JJ officials/NNS ./PUNCT"] ################################################################################ @@ -19,9 +19,9 @@ # - minimal number of words in the compression : 6 # - language of the input sentences : en (english) # - POS tag for punctuation marks : PUNCT -compresser = takahe.word_graph( sentences, - nb_words = 6, - lang = 'en', +compresser = takahe.word_graph( sentences, + nb_words = 6, + lang = 'en', punct_tag = "PUNCT" ) # Get the 50 best paths @@ -34,20 +34,20 @@ normalized_score = cummulative_score / len(path) # Print normalized score and compression - print round(normalized_score, 3), ' '.join([u[0] for u in path]) + print(round(normalized_score, 3), ' '.join([u[0] for u in path])) # Write the word graph in the dot format compresser.write_dot('test.dot') # 2. Rerank compressions by keyphrases (Boudin and Morin's method) -reranker = takahe.keyphrase_reranker( sentences, - candidates, +reranker = takahe.keyphrase_reranker( sentences, + candidates, lang = 'en' ) reranked_candidates = reranker.rerank_nbest_compressions() # Loop over the best reranked candidates for score, path in reranked_candidates: - + # Print the best reranked candidates - print round(score, 3), ' '.join([u[0] for u in path]) \ No newline at end of file + print(round(score, 3), ' '.join([u[0] for u in path])) diff --git a/takahe/__init__.pyc b/takahe/__init__.pyc deleted file mode 100644 index c6bf095..0000000 Binary files a/takahe/__init__.pyc and /dev/null differ diff --git a/takahe/takahe.py b/takahe/takahe.py index 109bacf..b1eaf40 100644 --- a/takahe/takahe.py +++ b/takahe/takahe.py @@ -15,22 +15,22 @@ Mar. 2013 :Description: - takahe is a multi-sentence compression module. Given a set of redundant - sentences, a word-graph is constructed by iteratively adding sentences to + takahe is a multi-sentence compression module. Given a set of redundant + sentences, a word-graph is constructed by iteratively adding sentences to it. The best compression is obtained by finding the shortest path in the word graph. The original algorithm was published and described in [filippova:2010:COLING]_. A keyphrase-based reranking method, described in - [boudin-morin:2013:NAACL]_ can be applied to generate more informative + [boudin-morin:2013:NAACL]_ can be applied to generate more informative compressions. - .. [filippova:2010:COLING] Katja Filippova, Multi-Sentence Compression: - Finding Shortest Paths in Word Graphs, *Proceedings of the 23rd - International Conference on Computational Linguistics (Coling 2010)*, + .. [filippova:2010:COLING] Katja Filippova, Multi-Sentence Compression: + Finding Shortest Paths in Word Graphs, *Proceedings of the 23rd + International Conference on Computational Linguistics (Coling 2010)*, pages 322-330, 2010. - .. [boudin-morin:2013:NAACL] Florian Boudin and Emmanuel Morin, Keyphrase - Extraction for N-best Reranking in Multi-Sentence Compression, + .. [boudin-morin:2013:NAACL] Florian Boudin and Emmanuel Morin, Keyphrase + Extraction for N-best Reranking in Multi-Sentence Compression, *Proceedings of the 2013 Conference of the North American Chapter of the - Association for Computational Linguistics: Human Language Technologies + Association for Computational Linguistics: Human Language Technologies (NAACL-HLT 2013)*, 2013. @@ -38,13 +38,13 @@ Development history of the takahe module: - 0.4 (Mar. 2013) adding the keyphrase-based nbest reranking algorithm - 0.33 (Feb. 2013), bug fixes and better code documentation - - 0.32 (Jun. 2012), Punctuation marks are now considered within the + - 0.32 (Jun. 2012), Punctuation marks are now considered within the graph, compressions are then punctuated - - 0.31 (Nov. 2011), modified context function (uses the left and right + - 0.31 (Nov. 2011), modified context function (uses the left and right contexts), improved docstring documentation, bug fixes - - 0.3 (Oct. 2011), improved K-shortest paths algorithm including + - 0.3 (Oct. 2011), improved K-shortest paths algorithm including verb/size constraints and ordered lists for performance - - 0.2 (Dec. 2010), removed dependencies from nltk (i.e. POS-tagging, + - 0.2 (Dec. 2010), removed dependencies from nltk (i.e. POS-tagging, tokenization and stopwords removal) - 0.1 (Nov. 2010), first version @@ -55,19 +55,19 @@ :Usage: A typical usage of this module is:: - + import takahe - + # A list of tokenized and POS-tagged sentences sentences = ['Hillary/NNP Clinton/NNP wanted/VBD to/stop visit/VB ...'] - + # Create a word graph from the set of sentences with parameters : # - minimal number of words in the compression : 6 # - language of the input sentences : en (english) # - POS tag for punctuation marks : PUNCT - compresser = takahe.word_graph( sentences, - nb_words = 6, - lang = 'en', + compresser = takahe.word_graph( sentences, + nb_words = 6, + lang = 'en', punct_tag = "PUNCT" ) # Get the 50 best paths @@ -86,23 +86,23 @@ compresser.write_dot('test.dot') # 2. Rerank compressions by keyphrases (Boudin and Morin's method) - reranker = takahe.keyphrase_reranker( sentences, - candidates, + reranker = takahe.keyphrase_reranker( sentences, + candidates, lang = 'en' ) reranked_candidates = reranker.rerank_nbest_compressions() # Loop over the best reranked candidates for score, path in reranked_candidates: - + # Print the best reranked candidates print round(score, 3), ' '.join([u[0] for u in path]) :Misc: The Takahe is a flightless bird indigenous to New Zealand. It was thought to - be extinct after the last four known specimens were taken in 1898. However, - after a carefully planned search effort the bird was rediscovered by on - November 20, 1948. (Wikipedia, http://en.wikipedia.org/wiki/takahe) + be extinct after the last four known specimens were taken in 1898. However, + after a carefully planned search effort the bird was rediscovered by on + November 20, 1948. (Wikipedia, http://en.wikipedia.org/wiki/takahe) """ import math @@ -121,16 +121,16 @@ class word_graph: """ The word_graph class constructs a word graph from the set of sentences given as input. The set of sentences is a list of strings, sentences are tokenized - and words are POS-tagged (e.g. ``"Saturn/NNP is/VBZ the/DT sixth/JJ - planet/NN from/IN the/DT Sun/NNP in/IN the/DT Solar/NNP System/NNP"``). + and words are POS-tagged (e.g. ``"Saturn/NNP is/VBZ the/DT sixth/JJ + planet/NN from/IN the/DT Sun/NNP in/IN the/DT Solar/NNP System/NNP"``). Four optional parameters can be specified: - - nb_words is is the minimal number of words for the best compression + - nb_words is is the minimal number of words for the best compression (default value is 8). - - lang is the language parameter and is used for selecting the correct - stopwords list (default is "en" for english, stopword lists are localized + - lang is the language parameter and is used for selecting the correct + stopwords list (default is "en" for english, stopword lists are localized in /resources/ directory). - - punct_tag is the punctuation mark tag used during graph construction + - punct_tag is the punctuation mark tag used during graph construction (default is PUNCT). """ @@ -142,7 +142,7 @@ def __init__(self, sentence_list, nb_words=8, lang="en", punct_tag="PUNCT", pos_ self.length = len(sentence_list) """ The number of sentences given for fusion. """ - + self.nb_words = nb_words """ The minimal number of words in the compression. """ @@ -163,7 +163,7 @@ def __init__(self, sentence_list, nb_words=8, lang="en", punct_tag="PUNCT", pos_ self.graph = nx.DiGraph() """ The directed graph used for fusion. """ - + self.start = '-start-' """ The start token in the graph. """ @@ -172,14 +172,14 @@ def __init__(self, sentence_list, nb_words=8, lang="en", punct_tag="PUNCT", pos_ self.sep = '/-/' """ The separator used between a word and its POS in the graph. """ - + self.term_freq = {} """ The frequency of a given term. """ - - self.verbs = set(['VB', 'VBD', 'VBP', 'VBZ', 'VH', 'VHD', 'VHP', 'VBZ', + + self.verbs = set(['VB', 'VBD', 'VBP', 'VBZ', 'VH', 'VHD', 'VHP', 'VBZ', 'VV', 'VVD', 'VVP', 'VVZ']) """ - The list of verb POS tags required in the compression. At least *one* + The list of verb POS tags required in the compression. At least *one* verb must occur in the candidate compressions. """ @@ -201,16 +201,16 @@ def __init__(self, sentence_list, nb_words=8, lang="en", punct_tag="PUNCT", pos_ #-T-----------------------------------------------------------------------T- def pre_process_sentences(self): """ - Pre-process the list of sentences given as input. Split sentences using + Pre-process the list of sentences given as input. Split sentences using whitespaces and convert each sentence to a list of (word, POS) tuples. """ for i in range(self.length): - + # Normalise extra white spaces self.sentence[i] = re.sub(' +', ' ', self.sentence[i]) self.sentence[i] = self.sentence[i].strip() - + # Tokenize the current sentence in word/POS sentence = self.sentence[i].split(' ') @@ -219,35 +219,35 @@ def pre_process_sentences(self): # Looping over the words for w in sentence: - + # Splitting word, POS pos_separator_re = re.escape(self.pos_separator) m = re.match("^(.+)" +pos_separator_re +"(.+)$", w) - + # Extract the word information token, POS = m.group(1), m.group(2) # Add the token/POS to the sentence container container.append((token.lower(), POS)) - + # Add the stop token at the end of the container container.append((self.stop, self.stop)) # Recopy the container into the current sentence self.sentence[i] = container #-B-----------------------------------------------------------------------B- - - + + #-T-----------------------------------------------------------------------T- def build_graph(self): """ Constructs a directed word graph from the list of input sentences. Each - sentence is iteratively added to the directed graph according to the + sentence is iteratively added to the directed graph according to the following algorithm: - Word mapping/creation is done in four steps: - 1. non-stopwords for which no candidate exists in the graph or for + 1. non-stopwords for which no candidate exists in the graph or for which an unambiguous mapping is possible or which occur more than once in the sentence @@ -258,22 +258,22 @@ def build_graph(self): 4. punctuation marks - For the last three groups of words where mapping is ambiguous we check - the immediate context (the preceding and following words in the sentence - and the neighboring nodes in the graph) and select the candidate which - has larger overlap in the context, or the one with a greater frequency - (i.e. the one which has more words mapped onto it). Stopwords are mapped - only if there is some overlap in non-stopwords neighbors, otherwise a - new node is created. Punctuation marks are mapped only if the preceding + For the last three groups of words where mapping is ambiguous we check + the immediate context (the preceding and following words in the sentence + and the neighboring nodes in the graph) and select the candidate which + has larger overlap in the context, or the one with a greater frequency + (i.e. the one which has more words mapped onto it). Stopwords are mapped + only if there is some overlap in non-stopwords neighbors, otherwise a + new node is created. Punctuation marks are mapped only if the preceding and following words in the sentence and the neighboring nodes are the same. - Edges are then computed and added between mapped words. - - Each node in the graph is represented as a tuple ('word/POS', id) and + + Each node in the graph is represented as a tuple ('word/POS', id) and possesses an info list containing (sentence_id, position_in_sentence) tuples. - """ + """ # Iteratively add each sentence in the graph --------------------------- for i in range(self.length): @@ -285,8 +285,8 @@ def build_graph(self): mapping = [0] * sentence_len #------------------------------------------------------------------- - # 1. non-stopwords for which no candidate exists in the graph or for - # which an unambiguous mapping is possible or which occur more + # 1. non-stopwords for which no candidate exists in the graph or for + # which an unambiguous mapping is possible or which occur more # than once in the sentence. #------------------------------------------------------------------- for j in range(sentence_len): @@ -297,7 +297,7 @@ def build_graph(self): # If stopword or punctuation mark, continues if token in self.stopwords or re.search('(?u)^\W$', token): continue - + # Create the node identifier node = token.lower() + self.sep + POS @@ -321,7 +321,7 @@ def build_graph(self): ids = [] for sid, pos_s in self.graph.node[(node, 0)]['info']: ids.append(sid) - + # Update the node in the graph if not same sentence if not i in ids: self.graph.node[(node, 0)]['info'].append((i, j)) @@ -329,7 +329,7 @@ def build_graph(self): # Else Create new node for redundant word else: - self.graph.add_node( (node, 1), info=[(i, j)], + self.graph.add_node( (node, 1), info=[(i, j)], label=token.lower() ) mapping[j] = (node, 1) @@ -341,7 +341,7 @@ def build_graph(self): # Get the word and tag token, POS = self.sentence[i][j] - + # If stopword or punctuation mark, continues if token in self.stopwords or re.search('(?u)^\W$', token): continue @@ -351,13 +351,13 @@ def build_graph(self): # Create the node identifier node = token.lower() + self.sep + POS - + # Create the neighboring nodes identifiers prev_token, prev_POS = self.sentence[i][j-1] next_token, next_POS = self.sentence[i][j+1] prev_node = prev_token.lower() + self.sep + prev_POS next_node = next_token.lower() + self.sep + next_POS - + # Find the number of ambiguous nodes in the graph k = self.ambiguous_nodes(node) @@ -365,16 +365,16 @@ def build_graph(self): # context or the greater frequency. ambinode_overlap = [] ambinode_frequency = [] - + # For each ambiguous node for l in range(k): # Get the immediate context words of the nodes l_context = self.get_directed_context(node, l, 'left') r_context = self.get_directed_context(node, l, 'right') - + # Compute the (directed) context sum - val = l_context.count(prev_node) + val = l_context.count(prev_node) val += r_context.count(next_node) # Add the count of the overlapping words @@ -384,36 +384,36 @@ def build_graph(self): ambinode_frequency.append( len( self.graph.node[(node, l)]['info'] ) ) - + # Search for the best candidate while avoiding a loop found = False selected = 0 while not found: - + # Select the ambiguous node selected = self.max_index(ambinode_overlap) if ambinode_overlap[selected] == 0: selected = self.max_index(ambinode_frequency) - + # Get the sentences id of this node ids = [] for sid, p in self.graph.node[(node, selected)]['info']: ids.append(sid) - + # Test if there is no loop if i not in ids: found = True break - + # Remove the candidate from the lists else: del ambinode_overlap[selected] del ambinode_frequency[selected] - + # Avoid endless loops if len(ambinode_overlap) == 0: break - + # Update the node in the graph if not same sentence if found: self.graph.node[(node, selected)]['info'].append((i, j)) @@ -421,10 +421,10 @@ def build_graph(self): # Else create new node for redundant word else: - self.graph.add_node( (node, k), info=[(i, j)], + self.graph.add_node( (node, k), info=[(i, j)], label=token.lower() ) mapping[j] = (node, k) - + #------------------------------------------------------------------- # 3. map the stopwords to the nodes #------------------------------------------------------------------- @@ -439,7 +439,7 @@ def build_graph(self): # Create the node identifier node = token.lower() + self.sep + POS - + # Find the number of ambiguous nodes in the graph k = self.ambiguous_nodes(node) @@ -447,15 +447,15 @@ def build_graph(self): if k == 0: # Add the node in the graph - self.graph.add_node( (node, 0), info=[(i, j)], + self.graph.add_node( (node, 0), info=[(i, j)], label=token.lower() ) # Mark the word as mapped to k mapping[j] = (node, 0) - + # Else find the node with overlap in context or create one else: - + # Create the neighboring nodes identifiers prev_token, prev_POS = self.sentence[i][j-1] next_token, next_POS = self.sentence[i][j+1] @@ -463,7 +463,7 @@ def build_graph(self): next_node = next_token.lower() + self.sep + next_POS ambinode_overlap = [] - + # For each ambiguous node for l in range(k): @@ -473,23 +473,23 @@ def build_graph(self): True) r_context = self.get_directed_context(node, l, 'right',\ True) - + # Compute the (directed) context sum - val = l_context.count(prev_node) + val = l_context.count(prev_node) val += r_context.count(next_node) # Add the count of the overlapping words ambinode_overlap.append(val) - + # Get best overlap candidate selected = self.max_index(ambinode_overlap) - + # Get the sentences id of the best candidate node ids = [] for sid, pos_s in self.graph.node[(node, selected)]['info']: ids.append(sid) - # Update the node in the graph if not same sentence and + # Update the node in the graph if not same sentence and # there is at least one overlap in context if i not in ids and ambinode_overlap[selected] > 0: # if i not in ids and \ @@ -525,7 +525,7 @@ def build_graph(self): # Create the node identifier node = token.lower() + self.sep + POS - + # Find the number of ambiguous nodes in the graph k = self.ambiguous_nodes(node) @@ -533,15 +533,15 @@ def build_graph(self): if k == 0: # Add the node in the graph - self.graph.add_node( (node, 0), info=[(i, j)], + self.graph.add_node( (node, 0), info=[(i, j)], label=token.lower() ) # Mark the word as mapped to k mapping[j] = (node, 0) - + # Else find the node with overlap in context or create one else: - + # Create the neighboring nodes identifiers prev_token, prev_POS = self.sentence[i][j-1] next_token, next_POS = self.sentence[i][j+1] @@ -549,30 +549,30 @@ def build_graph(self): next_node = next_token.lower() + self.sep + next_POS ambinode_overlap = [] - + # For each ambiguous node for l in range(k): # Get the immediate context words of the nodes l_context = self.get_directed_context(node, l, 'left') r_context = self.get_directed_context(node, l, 'right') - + # Compute the (directed) context sum - val = l_context.count(prev_node) + val = l_context.count(prev_node) val += r_context.count(next_node) # Add the count of the overlapping words ambinode_overlap.append(val) - + # Get best overlap candidate selected = self.max_index(ambinode_overlap) - + # Get the sentences id of the best candidate node ids = [] for sid, pos_s in self.graph.node[(node, selected)]['info']: ids.append(sid) - # Update the node in the graph if not same sentence and + # Update the node in the graph if not same sentence and # there is at least one overlap in context if i not in ids and ambinode_overlap[selected] > 1: @@ -585,7 +585,7 @@ def build_graph(self): # Else create a new node else: # Add the node in the graph - self.graph.add_node( (node, k), info=[(i, j)], + self.graph.add_node( (node, k), info=[(i, j)], label=token.lower() ) # Mark the word as mapped to k @@ -603,11 +603,11 @@ def build_graph(self): self.graph.add_edge(node1, node2, weight=edge_weight) #-B-----------------------------------------------------------------------B- - + #-T-----------------------------------------------------------------------T- def ambiguous_nodes(self, node): """ - Takes a node in parameter and returns the number of possible candidate + Takes a node in parameter and returns the number of possible candidate (ambiguous) nodes in the graph. """ k = 0 @@ -621,15 +621,15 @@ def ambiguous_nodes(self, node): def get_directed_context(self, node, k, dir='all', non_pos=False): """ Returns the directed context of a given node, i.e. a list of word/POS of - the left or right neighboring nodes in the graph. The function takes + the left or right neighboring nodes in the graph. The function takes four parameters : - node is the word/POS tuple - - k is the node identifier used when multiple nodes refer to the same + - k is the node identifier used when multiple nodes refer to the same word/POS (e.g. k=0 for (the/DET, 0), k=1 for (the/DET, 1), etc.) - - dir is the parameter that controls the directed context calculation, + - dir is the parameter that controls the directed context calculation, it can be set to left, right or all (default) - - non_pos is a boolean allowing to remove stopwords from the context + - non_pos is a boolean allowing to remove stopwords from the context (default is false) """ @@ -639,13 +639,13 @@ def get_directed_context(self, node, k, dir='all', non_pos=False): # For all the sentence/position tuples for sid, off in self.graph.node[(node, k)]['info']: - + prev = self.sentence[sid][off-1][0].lower() + self.sep +\ self.sentence[sid][off-1][1] - + next = self.sentence[sid][off+1][0].lower() + self.sep +\ self.sentence[sid][off+1][1] - + if non_pos: if self.sentence[sid][off-1][0] not in self.stopwords: l_context.append(prev) @@ -671,26 +671,26 @@ def get_directed_context(self, node, k, dir='all', non_pos=False): #-T-----------------------------------------------------------------------T- def get_edge_weight(self, node1, node2): """ - Compute the weight of an edge *e* between nodes *node1* and *node2*. It + Compute the weight of an edge *e* between nodes *node1* and *node2*. It is computed as e_ij = (A / B) / C with: - - - A = freq(i) + freq(j), + + - A = freq(i) + freq(j), - B = Sum (s in S) 1 / diff(s, i, j) - C = freq(i) * freq(j) - + A node is a tuple of ('word/POS', unique_id). """ # Get the list of (sentence_id, pos_in_sentence) for node1 info1 = self.graph.node[node1]['info'] - + # Get the list of (sentence_id, pos_in_sentence) for node2 info2 = self.graph.node[node2]['info'] - + # Get the frequency of node1 in the graph # freq1 = self.graph.degree(node1) freq1 = len(info1) - + # Get the frequency of node2 in cluster # freq2 = self.graph.degree(node2) freq2 = len(info2) @@ -700,34 +700,34 @@ def get_edge_weight(self, node1, node2): # For each sentence of the cluster (for s in S) for s in range(self.length): - + # Compute diff(s, i, j) which is calculated as # pos(s, i) - pos(s, j) if pos(s, i) < pos(s, j) # O otherwise - + # Get the positions of i and j in s, named pos(s, i) and pos(s, j) # As a word can appear at multiple positions in a sentence, a list # of positions is used pos_i_in_s = [] pos_j_in_s = [] - + # For each (sentence_id, pos_in_sentence) of node1 for sentence_id, pos_in_sentence in info1: # If the sentence_id is s if sentence_id == s: # Add the position in s pos_i_in_s.append(pos_in_sentence) - + # For each (sentence_id, pos_in_sentence) of node2 for sentence_id, pos_in_sentence in info2: # If the sentence_id is s if sentence_id == s: # Add the position in s pos_j_in_s.append(pos_in_sentence) - + # Container for all the diff(s, i, j) for i and j all_diff_pos_i_j = [] - + # Loop over all the i, j couples for x in range(len(pos_i_in_s)): for y in range(len(pos_j_in_s)): @@ -735,76 +735,76 @@ def get_edge_weight(self, node1, node2): # Test if word i appears *BEFORE* word j in s if diff_i_j < 0: all_diff_pos_i_j.append(-1.0*diff_i_j) - - # Add the mininum distance to diff (i.e. in case of multiple + + # Add the mininum distance to diff (i.e. in case of multiple # occurrencies of i or/and j in sentence s), 0 otherwise. if len(all_diff_pos_i_j) > 0: diff.append(1.0/min(all_diff_pos_i_j)) else: diff.append(0.0) - + weight1 = freq1 weight2 = freq2 return ( (freq1 + freq2) / sum(diff) ) / (weight1 * weight2) #-B-----------------------------------------------------------------------B- - - + + #-T-----------------------------------------------------------------------T- def k_shortest_paths(self, start, end, k=10): """ Simple implementation of a k-shortest paths algorithms. Takes three - parameters: the starting node, the ending node and the number of + parameters: the starting node, the ending node and the number of shortest paths desired. Returns a list of k tuples (path, weight). """ # Initialize the list of shortest paths kshortestpaths = [] - # Initializing the label container + # Initializing the label container orderedX = [] orderedX.append((0, start, 0)) - + # Initializing the path container paths = {} paths[(0, start, 0)] = [start] - + # Initialize the visited container visited = {} visited[start] = 0 - # Initialize the sentence container that will be used to remove + # Initialize the sentence container that will be used to remove # duplicate sentences passing throught different nodes sentence_container = {} - + # While the number of shortest paths isn't reached or all paths explored while len(kshortestpaths) < k and len(orderedX) > 0: - + # Searching for the shortest distance in orderedX shortest = orderedX.pop(0) shortestpath = paths[shortest] - + # Removing the shortest node from X and paths del paths[shortest] - + # Iterating over the accessible nodes for node in self.graph.neighbors(shortest[1]): - + # To avoid loops if node in shortestpath: continue - + # Compute the weight to node w = shortest[0] + self.graph[shortest[1]][node]['weight'] - - # If found the end, adds to k-shortest paths + + # If found the end, adds to k-shortest paths if node == end: #-T-------------------------------------------------------T- # --- Constraints on the shortest paths # 1. Check if path contains at least one werb - # 2. Check the length of the shortest path, without + # 2. Check the length of the shortest path, without # considering punctuation marks and starting node (-1 in # the range loop, because nodes are reversed) # 3. Check the paired parentheses and quotation marks @@ -834,7 +834,7 @@ def k_shortest_paths(self, start, end, k=10): quotation_mark_number += 1 # 4. raw_sentence += word + ' ' - + # Remove extra space from sentence raw_sentence = raw_sentence.strip() @@ -842,7 +842,7 @@ def k_shortest_paths(self, start, end, k=10): length >= self.nb_words and \ paired_parentheses == 0 and \ (quotation_mark_number%2) == 0 \ - and not sentence_container.has_key(raw_sentence): + and not raw_sentence in sentence_container: path = [node] path.extend(shortestpath) path.reverse() @@ -853,9 +853,9 @@ def k_shortest_paths(self, start, end, k=10): #-B-------------------------------------------------------B- else: - + # test if node has already been visited - if visited.has_key(node): + if node in visited: visited[node] += 1 else: visited[node] = 0 @@ -863,22 +863,22 @@ def k_shortest_paths(self, start, end, k=10): # Add the node to orderedX bisect.insort(orderedX, (w, node, id)) - + # Add the node to paths paths[(w, node, id)] = [node] paths[(w, node, id)].extend(shortestpath) - + # Returns the list of shortest paths return kshortestpaths #-B-----------------------------------------------------------------------B- - + #-T-----------------------------------------------------------------------T- def get_compression(self, nb_candidates=50): """ Searches all possible paths from **start** to **end** in the word graph, removes paths containing no verb or shorter than *n* words. Returns an - ordered list (smaller first) of nb (default value is 50) (cummulative - score, path) tuples. The score is not normalized with the sentence + ordered list (smaller first) of nb (default value is 50) (cummulative + score, path) tuples. The score is not normalized with the sentence length. """ @@ -889,15 +889,15 @@ def get_compression(self, nb_candidates=50): # Initialize the fusion container fusions = [] - + # Test if there are some paths if len(self.paths) > 0: - - # For nb candidates + + # For nb candidates for i in range(min(nb_candidates, len(self.paths))): nodes = self.paths[i][0] sentence = [] - + for j in range(1, len(nodes)-1): word, tag = nodes[j][0].split(self.sep) sentence.append((word, tag)) @@ -931,7 +931,7 @@ def compute_statistics(self): """ This function iterates over the cluster's sentences and computes the following statistics about each word: - + - term frequency (self.term_freq) """ @@ -940,15 +940,15 @@ def compute_statistics(self): # Loop over the sentences for i in range(self.length): - + # For each tuple (token, POS) of sentence i for token, POS in self.sentence[i]: - + # generate the word/POS token node = token.lower() + self.sep + POS - + # Add the token to the terms list - if not terms.has_key(node): + if not node in terms: terms[node] = [i] else: terms[node].append(i) @@ -964,7 +964,7 @@ def compute_statistics(self): #-T-----------------------------------------------------------------------T- def load_stopwords(self, path): """ - This function loads a stopword list from the *path* file and returns a + This function loads a stopword list from the *path* file and returns a set of words. Lines begining by '#' are ignored. """ @@ -979,12 +979,12 @@ def load_stopwords(self, path): # Return the set of stopwords return stopwords #-B-----------------------------------------------------------------------B- - + #-T-----------------------------------------------------------------------T- def write_dot(self, dotfile): """ Outputs the word graph in dot format in the specified file. """ - nx.write_dot(self.graph, dotfile) + nx.drawing.nx_pydot.write_dot(self.graph, dotfile) #-B-----------------------------------------------------------------------B- #~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~ @@ -998,33 +998,33 @@ def write_dot(self, dotfile): #~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~ class keyphrase_reranker: """ - The *keyphrase_reranker* reranks a list of compression candidates according - to the keyphrases they contain. Keyphrases are extracted from the set of - related sentences using a modified version of the TextRank method - [mihalcea-tarau:2004:EMNLP]_. First, an undirected weighted graph is - constructed from the set of sentences in which *nodes* are (lowercased word, + The *keyphrase_reranker* reranks a list of compression candidates according + to the keyphrases they contain. Keyphrases are extracted from the set of + related sentences using a modified version of the TextRank method + [mihalcea-tarau:2004:EMNLP]_. First, an undirected weighted graph is + constructed from the set of sentences in which *nodes* are (lowercased word, POS) tuples and *edges* represent co-occurrences. The TextRank algorithm is then applied on the graph to assign a score to each word. Second, keyphrase - candidates are extracted from the set of sentences using POS syntactic + candidates are extracted from the set of sentences using POS syntactic filtering. Keyphrases are then ranked according to the words they contain. - This class requires a set of related sentences (as a list of POS annotated - sentences) and the N-best compression candidates (as a list of (score, list - of (word, POS) tuples) tuples). The following optional parameters can be + This class requires a set of related sentences (as a list of POS annotated + sentences) and the N-best compression candidates (as a list of (score, list + of (word, POS) tuples) tuples). The following optional parameters can be specified: - - lang is the language parameter and is used for selecting the correct + - lang is the language parameter and is used for selecting the correct POS tags used for filtering keyphrase candidates. - - patterns is a list of extra POS patterns (regexes) used for filtering - keyphrase candidates, default is ``^(JJ)*(NNP|NNS|NN)+$`` for English and + - patterns is a list of extra POS patterns (regexes) used for filtering + keyphrase candidates, default is ``^(JJ)*(NNP|NNS|NN)+$`` for English and ``^(ADJ)*(NC|NPP)+(ADJ)*$`` for French. - .. [mihalcea-tarau:2004:EMNLP] Rada Mihalcea and Paul Tarau, TextRank: - Bringing Order into Texts, Empirical Methods in Natural Language + .. [mihalcea-tarau:2004:EMNLP] Rada Mihalcea and Paul Tarau, TextRank: + Bringing Order into Texts, Empirical Methods in Natural Language Processing (EMNLP), 2004. """ #-T-----------------------------------------------------------------------T- - def __init__(self, sentence_list, nbest_compressions, lang="en", + def __init__(self, sentence_list, nbest_compressions, lang="en", patterns=[], stopwords=[], pos_separator='/'): self.sentences = list(sentence_list) @@ -1090,17 +1090,17 @@ def __init__(self, sentence_list, nbest_compressions, lang="en", #-T-----------------------------------------------------------------------T- def build_graph(self, window=0): """ - Build a word graph from the list of sentences. Each node in the graph + Build a word graph from the list of sentences. Each node in the graph represents a word. An edge is created between two nodes if they co-occur in a given window (default is 0, indicating the whole sentence). """ - # For each sentence + # For each sentence for i in range(len(self.sentences)): - + # Normalise extra white spaces self.sentences[i] = re.sub(' +', ' ', self.sentences[i]) - + # Tokenize the current sentence in word/POS sentence = self.sentences[i].split(' ') @@ -1116,9 +1116,9 @@ def build_graph(self, window=0): # Modify the POS tags of stopwords to exclude them if sentence[j][0] in self.stopwords: - sentence[j] = (sentence[j][0], "STOPWORD") + sentence[j] = (sentence[j][0], "STOPWORD") - # Add the word only if it belongs to one of the syntactic + # Add the word only if it belongs to one of the syntactic # categories if sentence[j][1] in self.syntactic_filter: @@ -1133,7 +1133,7 @@ def build_graph(self, window=0): first_node = sentence[j] # Switch to set the window for the whole sentence - max_window = window + max_window = window if window < 1: max_window = len(sentence) @@ -1143,7 +1143,7 @@ def build_graph(self, window=0): # Get the second node second_node = sentence[k] - # Check if nodes exists + # Check if nodes exists if self.graph.has_node(first_node) and \ self.graph.has_node(second_node): @@ -1162,12 +1162,12 @@ def build_graph(self, window=0): #-T-----------------------------------------------------------------------T- def generate_candidates(self): """ - Function to generate the keyphrase candidates from the set of related + Function to generate the keyphrase candidates from the set of related sentences. Keyphrases candidates are the largest n-grams containing only words from the defined syntactic categories. """ - # For each sentence + # For each sentence for i in range(len(self.sentences)): sentence = self.sentences[i] @@ -1197,10 +1197,10 @@ def generate_candidates(self): candidate = [] else: - + # Flush the buffer candidate = [] - + # Handle the last possible candidate if len(candidate) > 0 and self.is_a_candidate(candidate): @@ -1213,7 +1213,7 @@ def generate_candidates(self): #-T-----------------------------------------------------------------------T- def is_a_candidate(self, keyphrase_candidate): """ - Function to check if a keyphrase candidate is a valid one according to + Function to check if a keyphrase candidate is a valid one according to the syntactic patterns. """ @@ -1230,15 +1230,15 @@ def is_a_candidate(self, keyphrase_candidate): #-T-----------------------------------------------------------------------T- def undirected_TextRank(self, d=0.85, f_conv=0.0001): """ - Implementation of the TextRank algorithm as described in + Implementation of the TextRank algorithm as described in [mihalcea-tarau:2004:EMNLP]_. Node scores are computed iteratively until - convergence (a threshold is used, default is 0.0001). The dampling + convergence (a threshold is used, default is 0.0001). The dampling factor is by default set to 0.85 as recommended in the article. """ # Initialise the maximum node difference for checking stability max_node_difference = f_conv - + # Initialise node scores to 1 self.word_scores = {} for node in self.graph.nodes(): @@ -1282,8 +1282,8 @@ def undirected_TextRank(self, d=0.85, f_conv=0.0001): #-T-----------------------------------------------------------------------T- def score_keyphrase_candidates(self): """ - Function to compute the score of each keyphrase candidate according to - the words it contains. The score of each keyphrase is calculated as the + Function to compute the score of each keyphrase candidate according to + the words it contains. The score of each keyphrase is calculated as the sum of its word scores normalized by its length + 1. """ @@ -1306,17 +1306,17 @@ def score_keyphrase_candidates(self): #-T-----------------------------------------------------------------------T- def cluster_keyphrase_candidates(self): """ - Function to cluster keyphrase candidates and remove redundancy. A large - number of the generated keyphrase candidates are redundant. Some + Function to cluster keyphrase candidates and remove redundancy. A large + number of the generated keyphrase candidates are redundant. Some keyphrases may be contained within larger ones, e.g. *giant tortoise* - and *Pinta Island giant tortoise*. To solve this problem, generated - keyphrases are clustered using word overlap. For each cluster, the + and *Pinta Island giant tortoise*. To solve this problem, generated + keyphrases are clustered using word overlap. For each cluster, the keyphrase with the highest score is selected. """ # Sort keyphrase candidates by length - descending = sorted(self.keyphrase_candidates, - key = lambda x: len(self.keyphrase_candidates[x]), + descending = sorted(self.keyphrase_candidates, + key = lambda x: len(self.keyphrase_candidates[x]), reverse=True) # Initialize the cluster container @@ -1326,7 +1326,7 @@ def cluster_keyphrase_candidates(self): for keyphrase in descending: found_cluster = False - + # Create a set of words from the keyphrase keyphrase_words = set(keyphrase.split(' ')) @@ -1339,7 +1339,7 @@ def cluster_keyphrase_candidates(self): # Check if keyphrase words are all contained in the cluster # representative words if len(keyphrase_words.difference(cluster_words)) == 0 : - + # Add keyphrase to cluster clusters[cluster].append(keyphrase) @@ -1357,8 +1357,8 @@ def cluster_keyphrase_candidates(self): for cluster in clusters: # Find the best scored keyphrase candidate in the cluster - sorted_cluster = sorted(clusters[cluster], - key=lambda cluster: self.keyphrase_scores[cluster], + sorted_cluster = sorted(clusters[cluster], + key=lambda cluster: self.keyphrase_scores[cluster], reverse=True) best_candidate_keyphrases.append(sorted_cluster[0]) @@ -1367,8 +1367,8 @@ def cluster_keyphrase_candidates(self): non_redundant_keyphrases = [] # Sort best candidate by score - sorted_keyphrases = sorted(best_candidate_keyphrases, - key=lambda keyphrase: self.keyphrase_scores[keyphrase], + sorted_keyphrases = sorted(best_candidate_keyphrases, + key=lambda keyphrase: self.keyphrase_scores[keyphrase], reverse=True) # Last loop to remove redundancy in cluster best candidates @@ -1382,7 +1382,7 @@ def cluster_keyphrase_candidates(self): non_redundant_keyphrases.append(keyphrase) # Modify the keyphrase candidate dictionnaries according to the clusters - for keyphrase in self.keyphrase_candidates.keys(): + for keyphrase in list(self.keyphrase_candidates.keys()): # Remove candidate if not in cluster if not keyphrase in non_redundant_keyphrases: @@ -1395,7 +1395,7 @@ def cluster_keyphrase_candidates(self): def rerank_nbest_compressions(self): """ Function that reranks the nbest compressions according to the keyphrases - they contain. The cummulative score (original score) is normalized by + they contain. The cummulative score (original score) is normalized by (compression length * Sum of keyphrase scores). """ @@ -1417,7 +1417,7 @@ def rerank_nbest_compressions(self): score = ( cummulative_score / (len(path) * total_keyphrase_score) ) - bisect.insort( reranked_compressions, + bisect.insort( reranked_compressions, (score, path) ) return reranked_compressions @@ -1437,7 +1437,7 @@ def wordpos_to_tuple(self, word): # Extract the word information token, POS = m.group(1), m.group(2) - # Return the tuple + # Return the tuple return (token.lower(), POS) #-B-----------------------------------------------------------------------B- @@ -1445,10 +1445,10 @@ def wordpos_to_tuple(self, word): #-T-----------------------------------------------------------------------T- def tuple_to_wordpos(self, wordpos_tuple): """ - This function converts a (word, POS) tuple to word/POS. The character + This function converts a (word, POS) tuple to word/POS. The character used for separating word and POS can be specified (default is /). """ - + # Return the word +delim+ POS return wordpos_tuple[0]+ self.pos_separator +wordpos_tuple[1] #-B-----------------------------------------------------------------------B- @@ -1457,4 +1457,3 @@ def tuple_to_wordpos(self, wordpos_tuple): #~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~ # ] Ending keyphrase_reranker class #~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~ -