BUGFIX: TMA-1643 Don't convert null value to empty string

Author: phong-nguyen-duy

lib/gooddata/cloud_resources/bigquery/bigquery_client.rb

@@ -60,10 +60,10 @@ def realize_query(query, _params)
             result = table_result.iterateAll
             field_list = table_result.getSchema.getFields
             col_count = field_list.size
-            CSV.open(filename, 'wb', :force_quotes => true) do |csv|
+            CSV.open(filename, 'wb') do |csv|
               csv << Array(1..col_count).map { |i| field_list.get(i - 1).getName } # build the header
               result.each do |row|
-                csv << Array(1..col_count).map { |i| row.get(i - 1).getStringValue }
+                csv << Array(1..col_count).map { |i| row.get(i - 1).getValue&.to_s }
               end
             end
           end

lib/gooddata/cloud_resources/redshift/redshift_client.rb

@@ -58,9 +58,9 @@ def realize_query(query, _params)
             result = statement.get_result_set
             metadata = result.get_meta_data
             col_count = metadata.column_count
-            CSV.open(filename, 'wb', :force_quotes => true) do |csv|
+            CSV.open(filename, 'wb') do |csv|
               csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
-              csv << Array(1..col_count).map { |i| result.get_string(i) } while result.next
+              csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
             end
           end
         end

lib/gooddata/cloud_resources/snowflake/snowflake_client.rb

@@ -54,9 +54,9 @@ def realize_query(query, _params)
             result = statement.get_result_set
             metadata = result.get_meta_data
             col_count = metadata.column_count
-            CSV.open(filename, 'wb', :force_quotes => true) do |csv|
+            CSV.open(filename, 'wb') do |csv|
               csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
-              csv << Array(1..col_count).map { |i| result.get_string(i) } while result.next
+              csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
             end
           end
         end

spec/data/bigquery_data.csv

@@ -1,5 +1,5 @@
-"EmployeeID","Fullname","Salary","DOB","Departments","x__client_id","x__timestamp","x__deleted"
-"E101","Name 01","4000","1970-01-20","101","mtt_testing_01","1557396000.001","false"
-"E102","Name 02","3000","1918-05-26","102","mtt_testing_01","1557396000.002","false"
-"E103","Name 03","5000","1960-02-03","103","mtt_testing_01","1557396000.003","false"
-"E104","Name 04","5000","1960-02-03","104","MTT-Client02","1557396000.004","false"
+EmployeeID,Fullname,Salary,DOB,Departments,x__client_id,x__timestamp,x__deleted
+E101,Name 01,4000,1970-01-20,101,mtt_testing_01,1557396000.0,false
+E102,Name 02,3000,1918-05-26,102,mtt_testing_01,1557396000.0,false
+E103,Name 03,5000,1960-02-03,103,mtt_testing_01,1557396000.0,false
+E104,"Name, 04",5000,,104,MTT-Client02,1557396000.0,false

spec/data/redshift_data.csv

@@ -1,3 +1,3 @@
-"login","client_id","role","user_groups"
-"nahunta@flsb11.com","client01","admin","GROUP_01"
-"rhabille@workwolf.site","client02","admin","GROUP_02"
+login,client_id,role,user_groups
+nahunta@flsb11.com,client01,admin,GROUP_01
+rhabille@workwolf.site,client02,admin,GROUP_02

spec/data/redshift_data2.csv

@@ -1,3 +1,3 @@
-"login","client_id","role","user_groups"
-"nahunta@flsb11.com","client01","admin","GROUP_01, GROUP_02"
-"rhabille@workwolf.site","client02","admin","GROUP_02"
+login,client_id,role,user_groups
+nahunta@flsb11.com,client01,admin,"GROUP_01, GROUP_02"
+rhabille@workwolf.site,client02,admin,GROUP_02

spec/data/snowflake_data.csv

@@ -1,2 +1,2 @@
-"CP__CUSTKEY","A__NAME","D__DATEADDED","X__TIMESTAMP"
-"cus3","phong","2019-03-05",""
+CP__CUSTKEY,A__NAME,D__DATEADDED,X__TIMESTAMP
+cus3,phong,,

spec/lcm/integration/data_helper_spec.rb

@@ -138,54 +138,61 @@
           }
       },
       "project"=> "gdc-us-dev",
-      "schema"=> "mtt_team_checklist"
+      "schema"=> "lcm_test"
     }
   },
   "input_source"=> {
     "type"=> "bigquery",
-    "query"=> "select * from employees;"
+    "query"=> "select * from employees order by EmployeeID"
   }
 }
+
 describe 'data helper', :vcr do
 
   it 'connect to redshift with IAM authentication' do
     data_helper = GoodData::Helpers::DataSource.new(iam_params['input_source'])
     file_path = data_helper.realize(iam_params)
+    puts "redshift iam: #{file_path}"
     data = File.open('spec/data/redshift_data2.csv').read
     expect(data).to eq File.open(file_path).read
   end
 
   it 'connect to redshift with BASIC authentication' do
     data_helper = GoodData::Helpers::DataSource.new(basic_params['input_source'])
     file_path = data_helper.realize(basic_params)
+    puts "redshift basic 1: #{file_path}"
     data = File.open('spec/data/redshift_data2.csv').read
     expect(data).to eq File.open(file_path).read
   end
 
   it 'connect to redshift with BASIC authentication without schema' do
     data_helper = GoodData::Helpers::DataSource.new(basic_params_without_schema['input_source'])
     file_path = data_helper.realize(basic_params_without_schema)
+    puts "redshift basic 2: #{file_path}"
     data = File.open('spec/data/redshift_data.csv').read
     expect(data).to eq File.open(file_path).read
   end
 
   it 'connect to redshift with BASIC authentication and dynamic source' do
     data_helper = GoodData::Helpers::DataSource.new(basic_params_dynamic_source['dynamic_params']['input_source'])
     file_path = data_helper.realize(basic_params_dynamic_source)
+    puts "redshift basic 3: #{file_path}"
     data = File.open('spec/data/redshift_data2.csv').read
     expect(data).to eq File.open(file_path).read
   end
 
   it 'connect to redshift with BASIC authentication and url has parameter' do
     data_helper = GoodData::Helpers::DataSource.new(basic_params_url_parameters['input_source'])
     file_path = data_helper.realize(basic_params_url_parameters)
+    puts "redshift basic 4: #{file_path}"
     data = File.open('spec/data/redshift_data2.csv').read
     expect(data).to eq File.open(file_path).read
   end
 
   it 'connect to snowflake with BASIC authentication' do
     data_helper = GoodData::Helpers::DataSource.new(snowflake_basic_params['input_source'])
     file_path = data_helper.realize(snowflake_basic_params)
+    puts "snowflake: #{file_path}"
     data = File.open('spec/data/snowflake_data.csv').read
     expect(data).to eq File.open(file_path).read
   end
@@ -198,6 +205,7 @@
 
     data_helper = GoodData::Helpers::DataSource.new(bigquery_basic_params['input_source'])
     file_path = data_helper.realize(bigquery_basic_params)
+    puts "bigquery: #{file_path}"
     data = File.open('spec/data/bigquery_data.csv').read
     expect(data).to eq File.open(file_path).read
   end