Skip to content

Commit 45cf8d5

Browse files
author
gdgate
authored
Merge pull request #1642 from phong-nguyen-duy/my-master
FEATURE: TMA-1630 Bump sdk version to 3.7.15 Reviewed-by: https://github.com/danh-ung
2 parents 2080eb6 + 4f7e6a4 commit 45cf8d5

23 files changed

Lines changed: 339 additions & 79 deletions

‎.rubocop.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ AllCops:
22
DisplayCopNames: true
33
Exclude:
44
- 'spec/helpers/appstore_project_helper.rb'
5+
TargetRubyVersion: 2.3
56

67
AbcSize:
78
Enabled: false

‎CHANGELOG.md‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,20 @@
11
# GoodData Ruby SDK Changelog
2+
## 2.1.9
3+
- FEATURE: TMA-1076 support new version api 2
4+
- BUGFIX: TMA-1637 handle input_source of dynamic params
5+
- BUGFIX: TMA-1636 Build csv file with force_quotes
6+
- FEATURE: TMA-1614 Support redshift input source
7+
- FEATURE: TMA-1259 Start using dataproduct in NFS release table
8+
- FEATURE: MSF-16455 support yellow WARNING status into RubySDK
9+
- CONFIG: TMA-1625 update version lock for test docker images
10+
- BUGFIX: TMA-1602 User filter brick failed - K8s bricks don't show error properly
11+
- BUGFIX: TMA-1593 Increase java heap space during execute bricks
12+
- BUGFIX: TMA-1558 K8s bricks don't show error when config invalid
13+
- BUGFIX: TMA-1596 The error propagated from a LCM brick is shown multiple times in the log
14+
- BUGFIX: TMA-1582 show synchronize ldm mode is running
15+
- FEATURE: TMA-1588 support schedule param include deprecated
16+
- FEATURE: TMA-1597 Logging lcm execution result
17+
218
## 2.1.8
319
- FEATURE: TMA-1604 Upgrade Restforce version to 3.x
420

‎VERSION‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
3.7.14
1+
3.7.15

‎lib/gooddata/cloud_resources/bigquery/bigquery_client.rb‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# encoding: UTF-8
2+
# frozen_string_literal: true
23
#
34
# Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
45
# This source code is licensed under the BSD-style license found in the
@@ -55,19 +56,19 @@ def realize_query(query, _params)
5556
query_config = QueryJobConfiguration.newBuilder(query).setDefaultDataset(@schema).build
5657
table_result = client.query(query_config)
5758

58-
if table_result.getTotalRows > 0
59+
if table_result.getTotalRows.positive?
5960
result = table_result.iterateAll
6061
field_list = table_result.getSchema.getFields
6162
col_count = field_list.size
62-
CSV.open(filename, 'wb', :force_quotes => true) do |csv|
63+
CSV.open(filename, 'wb') do |csv|
6364
csv << Array(1..col_count).map { |i| field_list.get(i - 1).getName } # build the header
6465
result.each do |row|
65-
csv << Array(1..col_count).map { |i| row.get(i - 1).getStringValue }
66+
csv << Array(1..col_count).map { |i| row.get(i - 1).getValue&.to_s }
6667
end
6768
end
6869
end
6970
end
70-
GoodData.gd_logger.info("Realize SQL query: type=redshift status=finished duration=#{measure.real}")
71+
GoodData.gd_logger.info("Realize SQL query: type=bigquery status=finished duration=#{measure.real}")
7172
filename
7273
end
7374

‎lib/gooddata/cloud_resources/redshift/redshift_client.rb‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# encoding: UTF-8
2+
# frozen_string_literal: true
23
#
34
# Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
45
# This source code is licensed under the BSD-style license found in the
@@ -57,9 +58,9 @@ def realize_query(query, _params)
5758
result = statement.get_result_set
5859
metadata = result.get_meta_data
5960
col_count = metadata.column_count
60-
CSV.open(filename, 'wb', :force_quotes => true) do |csv|
61+
CSV.open(filename, 'wb') do |csv|
6162
csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
62-
csv << Array(1..col_count).map { |i| result.get_string(i) } while result.next
63+
csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
6364
end
6465
end
6566
end

‎lib/gooddata/cloud_resources/snowflake/snowflake_client.rb‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# encoding: UTF-8
2+
# frozen_string_literal: true
23
#
34
# Copyright (c) 2010-2019 GoodData Corporation. All rights reserved.
45
# This source code is licensed under the BSD-style license found in the
@@ -53,16 +54,16 @@ def realize_query(query, _params)
5354
result = statement.get_result_set
5455
metadata = result.get_meta_data
5556
col_count = metadata.column_count
56-
CSV.open(filename, 'wb', :force_quotes => true) do |csv|
57+
CSV.open(filename, 'wb') do |csv|
5758
csv << Array(1..col_count).map { |i| metadata.get_column_name(i) } # build the header
58-
csv << Array(1..col_count).map { |i| result.get_string(i) } while result.next
59+
csv << Array(1..col_count).map { |i| result.get_string(i)&.to_s } while result.next
5960
end
6061
end
6162
end
6263
GoodData.gd_logger.info("Realize SQL query: type=snowflake status=finished duration=#{measure.real}")
6364
filename
6465
ensure
65-
@connection.close unless @connection.nil?
66+
@connection&.close
6667
@connection = nil
6768
end
6869

‎lib/gooddata/lcm/actions/collect_clients.rb‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -67,11 +67,11 @@ def call(params)
6767
end
6868

6969
def collect_clients(params, segment_names = nil)
70-
client_id_column = params.client_id_column || 'client_id'
71-
segment_id_column = params.segment_id_column || 'segment_id'
72-
project_id_column = params.project_id_column || 'project_id'
73-
project_title_column = params.project_title_column || 'project_title'
74-
project_token_column = params.project_token_column || 'project_token'
70+
client_id_column = params.client_id_column&.downcase || 'client_id'
71+
segment_id_column = params.segment_id_column&.downcase || 'segment_id'
72+
project_id_column = params.project_id_column&.downcase || 'project_id'
73+
project_title_column = params.project_title_column&.downcase || 'project_title'
74+
project_token_column = params.project_token_column&.downcase || 'project_token'
7575
client = params.gdc_gd_client
7676

7777
clients = []
@@ -82,7 +82,7 @@ def collect_clients(params, segment_names = nil)
8282
end
8383
GoodData.logger.debug("Input data: #{input_data.read}")
8484
GoodData.logger.debug("Segment names: #{segment_names}")
85-
CSV.foreach(input_data, :headers => true, :return_headers => false, encoding: 'utf-8') do |row|
85+
CSV.foreach(input_data, :headers => true, :return_headers => false, :header_converters => :downcase, :encoding => 'utf-8') do |row|
8686
GoodData.logger.debug("Processing row: #{row}")
8787
segment_name = row[segment_id_column]
8888
GoodData.logger.debug("Segment name: #{segment_name}")

‎lib/gooddata/lcm/actions/collect_dynamic_schedule_params.rb‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -38,11 +38,11 @@ class << self
3838
def call(params)
3939
return [] unless params.dynamic_params
4040

41-
schedule_title_column = params.schedule_title_column || 'schedule_title'
42-
client_id_column = params.client_id_column || 'client_id'
43-
param_name_column = params.param_name_column || 'param_name'
44-
param_value_column = params.param_value_column || 'param_value'
45-
param_secure_column = params.param_secure_column || 'param_secure'
41+
schedule_title_column = params.schedule_title_column&.downcase || 'schedule_title'
42+
client_id_column = params.client_id_column&.downcase || 'client_id'
43+
param_name_column = params.param_name_column&.downcase || 'param_name'
44+
param_value_column = params.param_value_column&.downcase || 'param_value'
45+
param_secure_column = params.param_secure_column&.downcase || 'param_secure'
4646

4747
encryption_key = params.dynamic_params_encryption_key || ''
4848
exist_encryption_key = encryption_key.blank? ? false : true
@@ -59,7 +59,7 @@ def call(params)
5959
schedule_hidden_params = {}
6060
exist_param_secure = false
6161

62-
CSV.foreach(input_data, :headers => true, :return_headers => false, encoding: 'utf-8') do |row|
62+
CSV.foreach(input_data, :headers => true, :return_headers => false, :header_converters => :downcase, :encoding => 'utf-8') do |row|
6363
is_param_secure = row[param_secure_column] == 'true'
6464
is_decrypt_secure_value = is_param_secure && exist_encryption_key ? true : false
6565
exist_param_secure = true if is_param_secure

‎lib/gooddata/lcm/actions/collect_users_brick_users.rb‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ class CollectUsersBrickUsers < BaseAction
3535
class << self
3636
def call(params)
3737
users_brick_users = []
38-
login_column = params.users_brick_config.login_column || 'login'
38+
login_column = params.users_brick_config.login_column&.downcase || 'login'
3939
users_brick_data_source = GoodData::Helpers::DataSource.new(params.users_brick_config.input_source)
4040

4141
users_brick_data_source_file = without_check(PARAMS, params) do
@@ -45,14 +45,15 @@ def call(params)
4545
)
4646
end
4747
CSV.foreach(users_brick_data_source_file,
48-
headers: true,
49-
return_headers: false,
50-
encoding: 'utf-8') do |row|
51-
pid = row[params.multiple_projects_column]
48+
:headers => true,
49+
:return_headers => false,
50+
:header_converters => :downcase,
51+
:encoding => 'utf-8') do |row|
52+
pid = row[params.multiple_projects_column&.downcase]
5253
fail "The set multiple_projects_column '#{params.multiple_projects_column}' of the users input is empty" if !pid && MULTIPLE_COLUMN_MODES.include?(params.sync_mode)
5354

5455
users_brick_users << {
55-
login: row[login_column].downcase,
56+
login: row[login_column].nil? ? nil : row[login_column].strip.downcase,
5657
pid: pid
5758
}
5859
end

‎lib/gooddata/lcm/actions/set_master_project.rb‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
# encoding: UTF-8
2+
# frozen_string_literal: true
3+
#
4+
# Copyright (c) 2010-2017 GoodData Corporation. All rights reserved.
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
require_relative 'base_action'
9+
10+
module GoodData
11+
module LCM2
12+
class SetMasterProject < BaseAction
13+
DESCRIPTION = 'Set master project'
14+
15+
PARAMS = define_params(self) do
16+
description 'Organization Name'
17+
param :organization, instance_of(Type::StringType), required: false
18+
19+
description 'Domain'
20+
param :domain, instance_of(Type::StringType), required: false
21+
22+
description 'ADS Client'
23+
param :ads_client, instance_of(Type::AdsClientType), required: false
24+
25+
description 'Table Name'
26+
param :release_table_name, instance_of(Type::StringType), required: false
27+
28+
description 'Segments to manage'
29+
param :segments, array_of(instance_of(Type::SegmentType)), required: true
30+
31+
description 'DataProduct to manage'
32+
param :data_product, instance_of(Type::GDDataProductType), required: false
33+
34+
description 'Released master project should be used in next rollout'
35+
param :set_master_project, instance_of(Type::StringType), required: false
36+
end
37+
38+
class << self
39+
def call(params)
40+
results = []
41+
domain_name = params.organization || params.domain
42+
data_product = params.data_product
43+
params.segments.each do |segment_in|
44+
version = get_latest_version(params, domain_name, data_product.data_product_id, segment_in.segment_id) + 1
45+
segment_in[:data_product_id] = data_product.data_product_id
46+
segment_in[:master_pid] = params.set_master_project
47+
segment_in[:version] = version
48+
segment_in[:timestamp] = Time.now.utc.iso8601
49+
50+
results << {
51+
data_product_id: data_product.data_product_id,
52+
segment_id: segment_in.segment_id,
53+
version: version
54+
}
55+
end
56+
results
57+
end
58+
59+
def get_latest_version(params, domain_name, data_product_id, segment_id)
60+
if params.ads_client
61+
current_master = GoodData::LCM2::Helpers.latest_master_project_from_ads(
62+
params.release_table_name,
63+
params.ads_client,
64+
segment_id
65+
)
66+
else
67+
current_master = GoodData::LCM2::Helpers.latest_master_project_from_nfs(domain_name, data_product_id, segment_id)
68+
end
69+
return 0 unless current_master
70+
71+
current_master[:version].to_i
72+
end
73+
end
74+
end
75+
end
76+
end

0 commit comments

Comments
 (0)