Merge pull request #78 from abeaumont/fix/incorrect-duplicated-tokens

Fix incorrect duplicated tokens

Author: abeaumont

driver/normalizer/parser.go

@@ -13,8 +13,7 @@ var ToNoder = &native.ObjectToNoder{
 	EndColumnKey:    "end_col_offset",
 
 	TokenKeys: map[string]bool{
-		"module": true, // Module on ImportFrom
-		"name":   true,
+		"name": true,
 		//"asname":          true, // Alias from ImportFrom
 		"id":           true, // Name nodes
 		"attr":         true, // something.attr
@@ -63,13 +62,13 @@ var ToNoder = &native.ObjectToNoder{
 		"Compare":  {"comparators": true, "ops": true},
 		// FIXME: check call.keywords
 		//"Call"                    : { "args": true, "keywords": true},
-		"With":                    {"body": true, "items": true},
-		"FunctionDef":             {"body": true, "decorator_list": true},
-		"Lambda":                  {"body": true},
-		"arguments":               {"defaults": true},
-		"Try":                     {"body": true, "orelse": true, "finalbody": true, "handlers": true},
-		"Raise":                   {"args": true},
-		"ClassDef":                {"body": true, "bases": true, "decorator_list": true, "keywords": true},
+		"With":        {"body": true, "items": true},
+		"FunctionDef": {"body": true, "decorator_list": true},
+		"Lambda":      {"body": true},
+		"arguments":   {"defaults": true},
+		"Try":         {"body": true, "orelse": true, "finalbody": true, "handlers": true},
+		"Raise":       {"args": true},
+		"ClassDef":    {"body": true, "bases": true, "decorator_list": true, "keywords": true},
 	},
 	PromotedPropertyStrings: map[string]map[string]bool{
 		"alias":      {"asname": true},

tests/import.py.uast

@@ -106,7 +106,6 @@ Module {
 .  .  }
 .  .  2: ImportFrom {
 .  .  .  Roles: ImportDeclaration,Statement
-.  .  .  TOKEN "os"
 .  .  .  StartPosition: {
 .  .  .  .  Offset: 31
 .  .  .  .  Line: 3
@@ -120,6 +119,7 @@ Module {
 .  .  .  Properties: {
 .  .  .  .  internalRole: body
 .  .  .  .  level: 0
+.  .  .  .  module: os
 .  .  .  }
 .  .  .  Children: {
 .  .  .  .  0: alias {
@@ -151,7 +151,6 @@ Module {
 .  .  }
 .  .  3: ImportFrom {
 .  .  .  Roles: ImportDeclaration,Statement
-.  .  .  TOKEN "os.path"
 .  .  .  StartPosition: {
 .  .  .  .  Offset: 46
 .  .  .  .  Line: 4
@@ -165,6 +164,7 @@ Module {
 .  .  .  Properties: {
 .  .  .  .  internalRole: body
 .  .  .  .  level: 0
+.  .  .  .  module: os.path
 .  .  .  }
 .  .  .  Children: {
 .  .  .  .  0: alias {

tests/issue62.py.uast

@@ -16,7 +16,6 @@ Module {
 .  Children: {
 .  .  0: ImportFrom {
 .  .  .  Roles: ImportDeclaration,Statement
-.  .  .  TOKEN "os"
 .  .  .  StartPosition: {
 .  .  .  .  Offset: 5
 .  .  .  .  Line: 1
@@ -30,6 +29,7 @@ Module {
 .  .  .  Properties: {
 .  .  .  .  internalRole: body
 .  .  .  .  level: 0
+.  .  .  .  module: os
 .  .  .  }
 .  .  .  Children: {
 .  .  .  .  0: alias {

tests/issue62_b.py.uast

@@ -16,7 +16,6 @@ Module {
 .  Children: {
 .  .  0: ImportFrom {
 .  .  .  Roles: ImportDeclaration,Statement
-.  .  .  TOKEN "collections"
 .  .  .  StartPosition: {
 .  .  .  .  Offset: 5
 .  .  .  .  Line: 1
@@ -30,6 +29,7 @@ Module {
 .  .  .  Properties: {
 .  .  .  .  internalRole: body
 .  .  .  .  level: 0
+.  .  .  .  module: collections
 .  .  .  }
 .  .  .  Children: {
 .  .  .  .  0: alias {
@@ -61,7 +61,6 @@ Module {
 .  .  }
 .  .  1: ImportFrom {
 .  .  .  Roles: ImportDeclaration,Statement
-.  .  .  TOKEN "ast2vec.bblfsh_roles"
 .  .  .  StartPosition: {
 .  .  .  .  Offset: 33
 .  .  .  .  Line: 3
@@ -75,6 +74,7 @@ Module {
 .  .  .  Properties: {
 .  .  .  .  internalRole: body
 .  .  .  .  level: 0
+.  .  .  .  module: ast2vec.bblfsh_roles
 .  .  .  }
 .  .  .  Children: {
 .  .  .  .  0: alias {
@@ -137,7 +137,6 @@ Module {
 .  .  }
 .  .  2: ImportFrom {
 .  .  .  Roles: ImportDeclaration,Statement
-.  .  .  TOKEN "ast2vec.repo2.base"
 .  .  .  StartPosition: {
 .  .  .  .  Offset: 84
 .  .  .  .  Line: 4
@@ -151,6 +150,7 @@ Module {
 .  .  .  Properties: {
 .  .  .  .  internalRole: body
 .  .  .  .  level: 0
+.  .  .  .  module: ast2vec.repo2.base
 .  .  .  }
 .  .  .  Children: {
 .  .  .  .  0: alias {