From f0383ec70f83ca2324c9d15607f7e6f9685ab0ad Mon Sep 17 00:00:00 2001 From: "akhilesh.trivedi" Date: Wed, 18 Mar 2020 13:58:12 +0530 Subject: [PATCH 01/19] initial code --- icons/Salesforce-batchsource.png | Bin 0 -> 972 bytes .../cdap/plugin/sfmc/DataExtensionClient.java | 857 +++++++++--------- .../sfmc/source/SalesforceInputFormat.java | 161 ++++ .../sfmc/source/SalesforceInputSplit.java | 73 ++ .../source/SalesforceJobConfiguration.java | 70 ++ .../sfmc/source/SalesforceRecordReader.java | 271 ++++++ .../plugin/sfmc/source/SalesforceSource.java | 101 +++ .../sfmc/source/SalesforceSourceConfig.java | 390 ++++++++ .../SalesforceTableAPIClientImpl.java | 93 ++ .../SalesforceTableDataResponse.java | 59 ++ .../ServiceNowTableAPIRequestBuilder.java | 75 ++ .../sfmc/source/util/SalesforceColumn.java | 54 ++ .../sfmc/source/util/SalesforceConstants.java | 93 ++ .../source/util/SalesforceObjectInfo.java | 46 + .../sfmc/source/util/SchemaBuilder.java | 90 ++ .../sfmc/source/util/SourceApplication.java | 80 ++ .../sfmc/source/util/SourceObjectMode.java | 65 ++ .../sfmc/source/util/SourceValueType.java | 71 ++ .../io/cdap/plugin/sfmc/source/util/Test.java | 99 ++ .../sfmc/source/util/TokenGenerator.java | 90 ++ .../io/cdap/plugin/sfmc/source/util/Util.java | 46 + widgets/Salesforce-batchsource.json | 151 +++ 22 files changed, 2617 insertions(+), 418 deletions(-) create mode 100644 icons/Salesforce-batchsource.png create mode 100644 src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java create mode 100644 src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputSplit.java create mode 100644 src/main/java/io/cdap/plugin/sfmc/source/SalesforceJobConfiguration.java create mode 100644 src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java create mode 100644 src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java create mode 100644 src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java create mode 100644 src/main/java/io/cdap/plugin/sfmc/source/apiclient/SalesforceTableAPIClientImpl.java create mode 100644 src/main/java/io/cdap/plugin/sfmc/source/apiclient/SalesforceTableDataResponse.java create mode 100644 src/main/java/io/cdap/plugin/sfmc/source/apiclient/ServiceNowTableAPIRequestBuilder.java create mode 100644 src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceColumn.java create mode 100644 src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java create mode 100644 src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceObjectInfo.java create mode 100644 src/main/java/io/cdap/plugin/sfmc/source/util/SchemaBuilder.java create mode 100644 src/main/java/io/cdap/plugin/sfmc/source/util/SourceApplication.java create mode 100644 src/main/java/io/cdap/plugin/sfmc/source/util/SourceObjectMode.java create mode 100644 src/main/java/io/cdap/plugin/sfmc/source/util/SourceValueType.java create mode 100644 src/main/java/io/cdap/plugin/sfmc/source/util/Test.java create mode 100644 src/main/java/io/cdap/plugin/sfmc/source/util/TokenGenerator.java create mode 100644 src/main/java/io/cdap/plugin/sfmc/source/util/Util.java create mode 100644 widgets/Salesforce-batchsource.json diff --git a/icons/Salesforce-batchsource.png b/icons/Salesforce-batchsource.png new file mode 100644 index 0000000000000000000000000000000000000000..46817a677bdf2dcf7e477aa7d76f8423e8467c9b GIT binary patch literal 972 zcmV;-12g=IP)}9>^518CPt>aI$<&V$q-|_k%pWZK~;J@VcpVjft>i7_t+^^m90Fv7O|Nnr= z?ZD*rV7ur*t>f_d{ScYlAfVsi^829H@ldqpFsI?L-Sp7w_>j==aKh@@gNI}Q000nl zQchC<5P+KSz_4&2;Lt!|Fz--tpbT~Z00Rg~L_t(&-tCy%mZKmLMOD-ojd}_cZ!yXL zf8`+BaaLMKo5qJ(%l#6ifMZcyDmW)6CnqN-Cp$>I;0nMGyu>*M*SliI_aetWo{y1P z#+^Ul7RFEnEO*R_Vtgk~CnNjNHY4@1gr5vvu+P$qJq=CnqzgWV2F^r2*6?qOw)p2N zF*dNEa)AMCTv&=Sd`BG{NHC zBAl9MXEa`JxLc&5JN6KM8?7rw3L*Le)k_G$t5KwEMc=(mB`4wqx?UHSyl6!?R1wb# zMK9X<1A-5Y+@FVA64eC_v|40*F2F|<*Osx6gKL|9ldAE+tg@D-2&z#lPl^gPkrt6k zpd_hG_rat@pQuO|Ql&&xRY@5N3@s3Hnwrv$P&{83({t~H%j)7`2@tPW-9^!k~(|}`)5TIj% zYH@Z&zzoqtsJ1OhA#_ai1Zo7eU#61!Y0*o1LW}4s4_OIS*Edp>s9Hfn za1^2dI-$x6lzl^@o4UnIkg|^zh9CLpYj7bO)p{Z^KiwbHVHcTc3S5;hlGq0`J$MVf z_V*E)iqxFnO&oD>2$^d;h1gEK0vrIdp9OE9R}rkkTnj0eLArf|Rj&Er8GW{PQxRp> z0)Cq$>imivt6W{j77S}7jy2rae7>5i*w@f1Y-wUSwz*|nw`KNk u?Zd=5h@1t3P|Plob8>QWa&mI=zwif=S_AZ*FkQF+0000 { - Collection columns = getDataExtensionInfo().getColumnList(); - if (columns == null || columns.isEmpty()) { - collector.addFailure(String.format("Data extension '%s' must exist.", dataExtensionKey), null) - .withConfigProperty(MarketingCloudConf.DATA_EXTENSION); - throw collector.getOrThrowException(); - } - Map lowercaseToOriginal = schema.getFields().stream() - .map(Schema.Field::getName) - .collect(Collectors.toMap(String::toLowerCase, f -> f)); - - for (ETDataExtensionColumn column : columns) { - String columnName = column.getName(); - String originalName = lowercaseToOriginal.get(columnName.toLowerCase()); - Schema.Field schemaField = schema.getField(originalName); - if (schemaField == null) { - if (column.getIsRequired()) { - collector.addFailure( - String.format("Data extension '%s' contains a required column '%s' of type '%s' that is not present in " + - "the input schema.", dataExtensionKey, columnName, column.getType()), null) - .withConfigProperty(MarketingCloudConf.DATA_EXTENSION); - } - } else { - Schema fieldSchema = schemaField.getSchema(); - if (fieldSchema.isNullable()) { - fieldSchema = fieldSchema.getNonNullable(); - } - Schema.Type schemaType = fieldSchema.getType(); - if (schemaType == Schema.Type.STRING) { - // Allow strings for anything, as the API just takes strings. - // If the value is malformed, it will fail at runtime. - continue; - } - String schemaTypeStr = fieldSchema.getLogicalType() != null ? - fieldSchema.getLogicalType().getToken() : schemaType.name().toLowerCase(); - String fieldName = schemaField.getName(); - switch (column.getType()) { - case BOOLEAN: - if (schemaType != Schema.Type.BOOLEAN) { - collector.addFailure( - String.format("Column '%s' is a boolean in data extension '%s', but is a '%s' in the input schema.", - originalName, dataExtensionKey, schemaTypeStr), - "Change the field schema to ensure it is a boolean or string type.") - .withInputSchemaField(fieldName); - } - break; - case DECIMAL: - if (fieldSchema.getLogicalType() != Schema.LogicalType.DECIMAL) { - collector.addFailure( - String.format("Column '%s' is a decimal in data extension '%s', but is a '%s' in the input schema.", - originalName, dataExtensionKey, schemaTypeStr), - "Change the field schema to ensure it is a decimal or string type.").withInputSchemaField(fieldName); - } - break; - case PHONE: - case TEXT: - case EMAIL_ADDRESS: - case LOCALE: - collector.addFailure( - String.format("Column '%s' is a %s in data extension '%s', but is a '%s' in the input schema.", - originalName, column.getType().name().toLowerCase(), dataExtensionKey, schemaTypeStr), - "Change the field schema to ensure it is a string type.").withInputSchemaField(fieldName); - break; - case DATE: - if (fieldSchema.getLogicalType() != Schema.LogicalType.DATE) { - collector.addFailure( - String.format("Column '%s' is a date in data extension '%s', but is a '%s' in the input schema.", - originalName, dataExtensionKey, schemaTypeStr), - "Change the field schema to ensure it is a date or string type.").withInputSchemaField(fieldName); - } - break; - case NUMBER: - if (schemaType != Schema.Type.INT) { - collector.addFailure( - String.format("Column '%s' is a number in data extension '%s', but is a '%s' in the input schema.", - originalName, dataExtensionKey, schemaTypeStr), - "Change the field schema to ensure it is an integer or string type.").withInputSchemaField(fieldName); - } - break; - default: - collector.addFailure( - String.format("Unknown type '%s' for column '%s' in data extension '%s'.", - column.getType(), column.getName(), dataExtensionKey), - "Supported types are: boolean, decimal, phone, text, email_address, locale, date and number.") - .withInputSchemaField(fieldName); - } - } - } - return null; - }); - } - - public DataExtensionInfo getDataExtensionInfo() throws ETSdkException { - return call(() -> { - List columns = ETDataExtension.retrieveColumns(client, dataExtensionKey); - return new DataExtensionInfo(dataExtensionKey, columns); - }); - } - - public List scan() throws ETSdkException { - return call(() -> ETDataExtension.select(client, "key=" + dataExtensionKey).getObjects()); - } - - public ETResponse insert(List rows) throws ETSdkException { - return call(() -> create(client, rows)); - } - - public ETResponse update(List rows) throws ETSdkException { - return call(() -> update(client, rows)); - } - - public List> upsert(List rows) throws ETSdkException { - ETResponse inserts = insert(rows); - List> result = new ArrayList<>(rows.size()); - - List toUpdate = new ArrayList<>(); - for (ETResult row : inserts.getResults()) { - // super hacky to check the error message... but there is no better way - if (row.getStatus() == ETResult.Status.ERROR && row.getErrorCode() == 2 && row.getErrorMessage() != null && - row.getErrorMessage().toLowerCase().contains("primary key")) { - ETDataExtensionRow failed = row.getObject(); - ETDataExtensionRow copy = new ETDataExtensionRow(); - copy.setDataExtensionKey(failed.getDataExtensionKey()); - for (String column : failed.getColumnNames()) { - copy.setColumn(column, failed.getColumn(column)); + + public static DataExtensionClient create(String dataExtensionKey, String clientId, String clientSecret, + String authEndpoint, String soapEndpoint) throws ETSdkException { + ETConfiguration conf = new ETConfiguration(); + conf.set("clientId", clientId); + conf.set("clientSecret", clientSecret); + conf.set("authEndpoint", authEndpoint); + conf.set("soapEndpoint", soapEndpoint); + conf.set("useOAuth2Authentication", "true"); + ClassLoader oldCL = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(DataExtensionClient.class.getClassLoader()); + return new DataExtensionClient(new ETClient(conf), dataExtensionKey); + } finally { + Thread.currentThread().setContextClassLoader(oldCL); } - toUpdate.add(copy); - } else { - result.add(row); - } } - if (!toUpdate.isEmpty()) { - ETResponse updates = update(client, toUpdate); - result.addAll(updates.getResults()); + public String getDataExtensionKey() { + return dataExtensionKey; } - return result; - } - - private T call(SFMCCall callable) throws ETSdkException { - ClassLoader oldClassloader = Thread.currentThread().getContextClassLoader(); - try { - Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); - return callable.call(); - } finally { - Thread.currentThread().setContextClassLoader(oldClassloader); - } - } - - /** - * A SFMC call - * - * @param type of return object - */ - private interface SFMCCall { /** - * Perform a call. + * Validate that the given schema is compatible with the given extension. + * + * @param schema the schema to check compatibility with + * @param collector failure collector + * @throws ETSdkException if there was an error getting the column information for the data extension */ - T call() throws ETSdkException; - } - - /** - * Copied from ETSoapObject.create(), except modified to include the error message. - */ - private static ETResponse create(ETClient client, - List objects) throws ETSdkException { - ETResponse response = new ETResponse<>(); - - if (objects == null || objects.size() == 0) { - response.setStatus(ETResult.Status.OK); - return response; + public void validateSchemaCompatibility(Schema schema, FailureCollector collector) throws ETSdkException { + call(() -> { + Collection columns = getDataExtensionInfo().getColumnList(); + if (columns == null || columns.isEmpty()) { + collector.addFailure(String.format("Data extension '%s' must exist.", + dataExtensionKey), null) + .withConfigProperty(MarketingCloudConf.DATA_EXTENSION); + throw collector.getOrThrowException(); + } + Map lowercaseToOriginal = schema.getFields().stream() + .map(Schema.Field::getName) + .collect(Collectors.toMap(String::toLowerCase, f -> f)); + + for (ETDataExtensionColumn column : columns) { + String columnName = column.getName(); + String originalName = lowercaseToOriginal.get(columnName.toLowerCase()); + Schema.Field schemaField = schema.getField(originalName); + if (schemaField == null) { + if (column.getIsRequired()) { + collector.addFailure( + String.format("Data extension '%s' contains a required column '%s' of type '%s'" + + " that is not present in " + + "the input schema.", dataExtensionKey, columnName, column.getType()) + , null) + .withConfigProperty(MarketingCloudConf.DATA_EXTENSION); + } + } else { + Schema fieldSchema = schemaField.getSchema(); + if (fieldSchema.isNullable()) { + fieldSchema = fieldSchema.getNonNullable(); + } + Schema.Type schemaType = fieldSchema.getType(); + if (schemaType == Schema.Type.STRING) { + // Allow strings for anything, as the API just takes strings. + // If the value is malformed, it will fail at runtime. + continue; + } + String schemaTypeStr = fieldSchema.getLogicalType() != null ? + fieldSchema.getLogicalType().getToken() : schemaType.name().toLowerCase(); + String fieldName = schemaField.getName(); + switch (column.getType()) { + case BOOLEAN: + if (schemaType != Schema.Type.BOOLEAN) { + collector.addFailure( + String.format("Column '%s' is a boolean in data extension '%s', but is a '%s'" + + " in the input schema.", + originalName, dataExtensionKey, schemaTypeStr), + "Change the field schema to ensure it is" + + " a boolean or string type.") + .withInputSchemaField(fieldName); + } + break; + case DECIMAL: + if (fieldSchema.getLogicalType() != Schema.LogicalType.DECIMAL) { + collector.addFailure( + String.format("Column '%s' is a decimal in data extension '%s'," + + " but is a '%s' in the input schema.", + originalName, dataExtensionKey, schemaTypeStr), + "Change the field schema to ensure it is a decimal" + + " or string type.").withInputSchemaField(fieldName); + } + break; + case PHONE: + case TEXT: + case EMAIL_ADDRESS: + case LOCALE: + collector.addFailure( + String.format("Column '%s' is a %s in data extension '%s', but is a '%s'" + + " in the input schema.", + originalName, column.getType().name().toLowerCase(), dataExtensionKey, + schemaTypeStr), + "Change the field schema to ensure it is a string type.") + .withInputSchemaField(fieldName); + break; + case DATE: + if (fieldSchema.getLogicalType() != Schema.LogicalType.DATE) { + collector.addFailure( + String.format("Column '%s' is a date in data extension '%s', but is a '%s'" + + " in the input schema.", + originalName, dataExtensionKey, schemaTypeStr), + "Change the field schema to ensure it is a date or string type." + ).withInputSchemaField(fieldName); + } + break; + case NUMBER: + if (schemaType != Schema.Type.INT) { + collector.addFailure( + String.format("Column '%s' is a number in data extension '%s', but is a '%s'" + + " in the input schema.", + originalName, dataExtensionKey, schemaTypeStr), + "Change the field schema to ensure it is an integer or string" + + " type.").withInputSchemaField(fieldName); + } + break; + default: + collector.addFailure( + String.format("Unknown type '%s' for column '%s' in data extension '%s'.", + column.getType(), column.getName(), dataExtensionKey), + "Supported types are: boolean, decimal, phone, text, email_address," + + " locale, date and number.") + .withInputSchemaField(fieldName); + } + } + } + return null; + }); } - // - // Get handle to the SOAP connection: - // + public DataExtensionInfo getDataExtensionInfo() throws ETSdkException { + return call(() -> { + List columns = ETDataExtension.retrieveColumns(client, dataExtensionKey); + return new DataExtensionInfo(dataExtensionKey, columns); + }); + } - ETSoapConnection connection = client.getSoapConnection(); + public List scan() throws ETSdkException { + return call(() -> ETDataExtension.select(client, "key=" + dataExtensionKey).getObjects()); + } - // - // Automatically refresh the token if necessary: - // + public ETResponse insert(List rows) throws ETSdkException { + return call(() -> create(client, rows)); + } - client.refreshToken(); + public ETResponse update(List rows) throws ETSdkException { + return call(() -> update(client, rows)); + } + + public List> upsert(List rows) throws ETSdkException { + ETResponse inserts = insert(rows); + List> result = new ArrayList<>(rows.size()); + + List toUpdate = new ArrayList<>(); + for (ETResult row : inserts.getResults()) { + // super hacky to check the error message... but there is no better way + if (row.getStatus() == ETResult.Status.ERROR && row.getErrorCode() == 2 && row.getErrorMessage() != null && + row.getErrorMessage().toLowerCase().contains("primary key")) { + ETDataExtensionRow failed = row.getObject(); + ETDataExtensionRow copy = new ETDataExtensionRow(); + copy.setDataExtensionKey(failed.getDataExtensionKey()); + for (String column : failed.getColumnNames()) { + copy.setColumn(column, failed.getColumn(column)); + } + toUpdate.add(copy); + } else { + result.add(row); + } + } - // - // Perform the SOAP create: - // - String obj = ""; + if (!toUpdate.isEmpty()) { + ETResponse updates = update(client, toUpdate); + result.addAll(updates.getResults()); + } + return result; + } - CreateRequest createRequest = new CreateRequest(); - createRequest.setOptions(new CreateOptions()); - for (T object : objects) { - object.setClient(client); - createRequest.getObjects().add(object.toInternal()); - obj += object.getClass().getSimpleName().substring(2); + private T call(SFMCCall callable) throws ETSdkException { + ClassLoader oldClassloader = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); + return callable.call(); + } finally { + Thread.currentThread().setContextClassLoader(oldClassloader); + } } - Soap soap = connection.getSoap("create", obj); - CreateResponse createResponse = soap.create(createRequest); + /** + * A SFMC call + * + * @param type of return object + */ + private interface SFMCCall { - response.setRequestId(createResponse.getRequestID()); - if (createResponse.getOverallStatus().equals("OK")) { - response.setStatus(ETResult.Status.OK); - } else if (createResponse.getOverallStatus().equals("Error")) { - response.setStatus(ETResult.Status.ERROR); - } - response.setResponseCode(createResponse.getOverallStatus()); - response.setResponseMessage(createResponse.getOverallStatus()); - for (CreateResult createResult : createResponse.getResults()) { - // - // Allocate a new (external) object: - // - - @SuppressWarnings("unchecked") - Class externalType = (Class) objects.get(0).getClass(); - - T externalObject = null; - try { - externalObject = externalType.newInstance(); - } catch (Exception ex) { - throw new ETSdkException("could not instantiate " + externalType.getName(), ex); - } - - externalObject.setClient(client); - - // - // Convert from internal representation: - // - - // not all SOAP calls return the object though some do.. - APIObject internalObject = createResult.getObject(); - if (internalObject != null) { - externalObject.fromInternal(createResult.getObject()); - } else { - // XXX populate fields from the object passed to the call? - externalObject.setId(Integer.toString(createResult.getNewID())); - } - - // - // Add result to the list of results: - // - - ETResult result = new ETResult(); - if (createResult.getStatusCode().equals("OK")) { - result.setStatus(ETResult.Status.OK); - } else if (createResult.getStatusCode().equals("Error")) { - result.setStatus(ETResult.Status.ERROR); - } - result.setResponseCode(createResult.getStatusCode()); - result.setResponseMessage(createResult.getStatusMessage()); - result.setErrorCode(createResult.getErrorCode()); - - // this is modified, cast to DataExtensionCreateResult since it has the error message - if (createResult instanceof DataExtensionCreateResult) { - DataExtensionCreateResult extCreateResult = (DataExtensionCreateResult) createResult; - result.setErrorMessage(extCreateResult.getErrorMessage()); - } - - // this is also modified, so we know the object that failed - //if (result.getResponseCode().equals("OK")) { // XXX? - result.setObject(externalObject); - //} - response.addResult(result); + /** + * Perform a call. + */ + T call() throws ETSdkException; } - return response; - } + /** + * Copied from ETSoapObject.create(), except modified to include the error message. + */ + private static ETResponse create(ETClient client, + List objects) throws ETSdkException { + ETResponse response = new ETResponse<>(); - /** - * Copy of ETSoapObject.update() except with modifications to set error message and original object. - */ - public static ETResponse update(ETClient client, - List objects) throws ETSdkException { - ETResponse response = new ETResponse<>(); + if (objects == null || objects.size() == 0) { + response.setStatus(ETResult.Status.OK); + return response; + } - if (objects == null || objects.size() == 0) { - response.setStatus(ETResult.Status.OK); - return response; - } + // + // Get handle to the SOAP connection: + // + + ETSoapConnection connection = client.getSoapConnection(); - // - // Get handle to the SOAP connection: - // + // + // Automatically refresh the token if necessary: + // - ETSoapConnection connection = client.getSoapConnection(); + client.refreshToken(); - // - // Automatically refresh the token if necessary: - // + // + // Perform the SOAP create: + // + String obj = ""; - client.refreshToken(); + CreateRequest createRequest = new CreateRequest(); + createRequest.setOptions(new CreateOptions()); + for (T object : objects) { + object.setClient(client); + createRequest.getObjects().add(object.toInternal()); + obj += object.getClass().getSimpleName().substring(2); + } + Soap soap = connection.getSoap("create", obj); + + CreateResponse createResponse = soap.create(createRequest); - // - // Perform the SOAP update: - // - String obj = ""; + response.setRequestId(createResponse.getRequestID()); + if (createResponse.getOverallStatus().equals("OK")) { + response.setStatus(ETResult.Status.OK); + } else if (createResponse.getOverallStatus().equals("Error")) { + response.setStatus(ETResult.Status.ERROR); + } + response.setResponseCode(createResponse.getOverallStatus()); + response.setResponseMessage(createResponse.getOverallStatus()); + for (CreateResult createResult : createResponse.getResults()) { + // + // Allocate a new (external) object: + // + + @SuppressWarnings("unchecked") + Class externalType = (Class) objects.get(0).getClass(); + + T externalObject = null; + try { + externalObject = externalType.newInstance(); + } catch (Exception ex) { + throw new ETSdkException("could not instantiate " + externalType.getName(), ex); + } + + externalObject.setClient(client); + + // + // Convert from internal representation: + // + + // not all SOAP calls return the object though some do.. + APIObject internalObject = createResult.getObject(); + if (internalObject != null) { + externalObject.fromInternal(createResult.getObject()); + } else { + // XXX populate fields from the object passed to the call? + externalObject.setId(Integer.toString(createResult.getNewID())); + } + + // + // Add result to the list of results: + // + + ETResult result = new ETResult(); + if (createResult.getStatusCode().equals("OK")) { + result.setStatus(ETResult.Status.OK); + } else if (createResult.getStatusCode().equals("Error")) { + result.setStatus(ETResult.Status.ERROR); + } + result.setResponseCode(createResult.getStatusCode()); + result.setResponseMessage(createResult.getStatusMessage()); + result.setErrorCode(createResult.getErrorCode()); + + // this is modified, cast to DataExtensionCreateResult since it has the error message + if (createResult instanceof DataExtensionCreateResult) { + DataExtensionCreateResult extCreateResult = (DataExtensionCreateResult) createResult; + result.setErrorMessage(extCreateResult.getErrorMessage()); + } + + // this is also modified, so we know the object that failed + //if (result.getResponseCode().equals("OK")) { // XXX? + result.setObject(externalObject); + //} + response.addResult(result); + } - UpdateRequest updateRequest = new UpdateRequest(); - updateRequest.setOptions(new UpdateOptions()); - for (ETDataExtensionRow object : objects) { - object.setClient(client); - updateRequest.getObjects().add(object.toInternal()); - obj += object.getClass().getSimpleName().substring(2); + return response; } - Soap soap = connection.getSoap("update", obj); - UpdateResponse updateResponse = soap.update(updateRequest); + /** + * Copy of ETSoapObject.update() except with modifications to set error message and original object. + */ + public static ETResponse update(ETClient client, + List objects) throws ETSdkException { + ETResponse response = new ETResponse<>(); - response.setRequestId(updateResponse.getRequestID()); - if (updateResponse.getOverallStatus().equals("OK")) { - response.setStatus(ETResult.Status.OK); - } else if (updateResponse.getOverallStatus().equals("Error")) { - response.setStatus(ETResult.Status.ERROR); - } - response.setResponseCode(updateResponse.getOverallStatus()); - response.setResponseMessage(updateResponse.getOverallStatus()); - for (UpdateResult updateResult : updateResponse.getResults()) { - // - // Allocate a new (external) object: - // - - @SuppressWarnings("unchecked") - Class externalType = (Class) objects.get(0).getClass(); - - ETDataExtensionRow externalObject = null; - try { - externalObject = externalType.newInstance(); - } catch (Exception ex) { - throw new ETSdkException("could not instantiate " - + externalType.getName(), ex); - } - - externalObject.setClient(client); - - // - // Convert from internal representation: - // - - // not all SOAP calls return the object though some do.. - APIObject internalObject = updateResult.getObject(); - if (internalObject != null) { - externalObject.fromInternal(updateResult.getObject()); - } else { - // XXX populate fields from the object passed to the call? - } - - // - // Add result to the list of results: - // - - ETResult result = new ETResult<>(); - if (updateResult.getStatusCode().equals("OK")) { - result.setStatus(ETResult.Status.OK); - } else if (updateResult.getStatusCode().equals("Error")) { - result.setStatus(ETResult.Status.ERROR); - } - result.setResponseCode(updateResult.getStatusCode()); - result.setResponseMessage(updateResult.getStatusMessage()); - result.setErrorCode(updateResult.getErrorCode()); - - - // this is modified, cast to DataExtensionCreateResult since it has the error message - if (updateResult instanceof DataExtensionUpdateResult) { - DataExtensionUpdateResult extUpdateResult = (DataExtensionUpdateResult) updateResult; - result.setErrorMessage(extUpdateResult.getErrorMessage()); - } - - // this is also modified, so we know the object that failed - //if (result.getResponseCode().equals("OK")) { // XXX? - result.setObject(externalObject); - //} - - if (result.getResponseCode().equals("OK")) { // XXX? - result.setObject(externalObject); - } - response.addResult(result); + if (objects == null || objects.size() == 0) { + response.setStatus(ETResult.Status.OK); + return response; + } + + // + // Get handle to the SOAP connection: + // + + ETSoapConnection connection = client.getSoapConnection(); + + // + // Automatically refresh the token if necessary: + // + + client.refreshToken(); + + // + // Perform the SOAP update: + // + String obj = ""; + + UpdateRequest updateRequest = new UpdateRequest(); + updateRequest.setOptions(new UpdateOptions()); + for (ETDataExtensionRow object : objects) { + object.setClient(client); + updateRequest.getObjects().add(object.toInternal()); + obj += object.getClass().getSimpleName().substring(2); + } + Soap soap = connection.getSoap("update", obj); + + UpdateResponse updateResponse = soap.update(updateRequest); + + response.setRequestId(updateResponse.getRequestID()); + if (updateResponse.getOverallStatus().equals("OK")) { + response.setStatus(ETResult.Status.OK); + } else if (updateResponse.getOverallStatus().equals("Error")) { + response.setStatus(ETResult.Status.ERROR); + } + response.setResponseCode(updateResponse.getOverallStatus()); + response.setResponseMessage(updateResponse.getOverallStatus()); + for (UpdateResult updateResult : updateResponse.getResults()) { + // + // Allocate a new (external) object: + // + + @SuppressWarnings("unchecked") + Class externalType = (Class) objects.get(0).getClass(); + + ETDataExtensionRow externalObject = null; + try { + externalObject = externalType.newInstance(); + } catch (Exception ex) { + throw new ETSdkException("could not instantiate " + + externalType.getName(), ex); + } + + externalObject.setClient(client); + + // + // Convert from internal representation: + // + + // not all SOAP calls return the object though some do.. + APIObject internalObject = updateResult.getObject(); + if (internalObject != null) { + externalObject.fromInternal(updateResult.getObject()); + } else { + // XXX populate fields from the object passed to the call? + } + + // + // Add result to the list of results: + // + + ETResult result = new ETResult<>(); + if (updateResult.getStatusCode().equals("OK")) { + result.setStatus(ETResult.Status.OK); + } else if (updateResult.getStatusCode().equals("Error")) { + result.setStatus(ETResult.Status.ERROR); + } + result.setResponseCode(updateResult.getStatusCode()); + result.setResponseMessage(updateResult.getStatusMessage()); + result.setErrorCode(updateResult.getErrorCode()); + + + // this is modified, cast to DataExtensionCreateResult since it has the error message + if (updateResult instanceof DataExtensionUpdateResult) { + DataExtensionUpdateResult extUpdateResult = (DataExtensionUpdateResult) updateResult; + result.setErrorMessage(extUpdateResult.getErrorMessage()); + } + + // this is also modified, so we know the object that failed + //if (result.getResponseCode().equals("OK")) { // XXX? + result.setObject(externalObject); + //} + + if (result.getResponseCode().equals("OK")) { // XXX? + result.setObject(externalObject); + } + response.addResult(result); + } + + return response; } - return response; - } - - /** - * Used to run the client from the command line. Primarily used for testing purposes. - * Expects arguments in the form [command] [key] [properties files] where properties file is expected - * to contain lines like: - * - * clientId=... - * clientSecret=... - * authEndpoint=... - * soapEndpoint=... - */ - public static void main(String[] args) throws Exception { - String command = args[0]; + /** + * Used to run the client from the command line. Primarily used for testing purposes. + * Expects arguments in the form [command] [key] [properties files] where properties file is expected + * to contain lines like: + *

+ * clientId=... + * clientSecret=... + * authEndpoint=... + * soapEndpoint=... + */ + public static void main(String[] args) throws Exception { + /* String command = args[0]; String key = args[1]; - String propertiesFile = args[2]; - ETConfiguration conf = new ETConfiguration(propertiesFile); - - DataExtensionClient client = new DataExtensionClient(new ETClient(conf), key); - - if ("describe".equals(command)) { - Collection columns = client.getDataExtensionInfo().getColumnList(); - System.out.println(columns.stream().map(c -> c.getName() + " " + c.getType()).collect(Collectors.joining("\n"))); - } else if ("scan".equals(command)) { - Collection columns = client.getDataExtensionInfo().getColumnList(); - List columnNames = columns.stream().map(ETDataExtensionColumn::getName).collect(Collectors.toList()); - System.out.println(String.join(", ", columnNames)); - for (ETDataExtensionRow row : client.scan()) { - List values = new ArrayList<>(columnNames.size()); - for (String columnName : columnNames) { - values.add(row.getColumn(columnName)); + String propertiesFile = args[2];*/ + String command = "describe"; + // String key ="2E2DF3B8-F3A0-415C-9EFC-922D32E13E61"; + String key = "7CBE457F-F056-4154-B21F-647CD11B81CF"; + ETConfiguration conf = new ETConfiguration("/Users/akhilesh.trivedi/gitcdap/FuelSDK-Java-Example" + + "/src/main/java/com/exacttarget/fuelsdk/test/fuelsdk.properties"); + + DataExtensionClient client = new DataExtensionClient(new ETClient(conf), key); + + if ("describe".equals(command)) { + Collection columns = client.getDataExtensionInfo().getColumnList(); + System.out.println(columns.stream().map(c -> c.getName() + " " + c.getType()) + .collect(Collectors.joining("\n"))); + } else if ("scan".equals(command)) { + Collection columns = client.getDataExtensionInfo().getColumnList(); + List columnNames = columns.stream().map(ETDataExtensionColumn::getName) + .collect(Collectors.toList()); + System.out.println(String.join(", ", columnNames)); + for (ETDataExtensionRow row : client.scan()) { + List values = new ArrayList<>(columnNames.size()); + for (String columnName : columnNames) { + values.add(row.getColumn(columnName)); + } + System.out.println(String.join(", ", values)); + } } - System.out.println(String.join(", ", values)); - } } - } } diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java new file mode 100644 index 0000000..f1f3eea --- /dev/null +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java @@ -0,0 +1,161 @@ +/* + * Copyright © 2017-2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.sfmc.source; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sfmc.source.apiclient.SalesforceTableAPIClientImpl; + +import io.cdap.plugin.sfmc.source.apiclient.SalesforceTableDataResponse; +import io.cdap.plugin.sfmc.source.util.SalesforceColumn; +import io.cdap.plugin.sfmc.source.util.SalesforceObjectInfo; +import io.cdap.plugin.sfmc.source.util.SchemaBuilder; +import io.cdap.plugin.sfmc.source.util.SourceObjectMode; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapreduce.InputFormat; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PAGE_SIZE; + +/** + * ServiceNow input format + */ +public class SalesforceInputFormat extends InputFormat { + private static final Logger LOG = LoggerFactory.getLogger(SalesforceInputFormat.class); + + /** + * Configure the input format to read tables from ServiceNow. Should be called from the mapreduce client. + * + * @param jobConfig the job configuration + * @param mode + * @param conf the database conf + * @return Collection of SalesforceObjectInfo containing table and schema. + */ + public static List setInput(Configuration jobConfig, SourceObjectMode mode, + SalesforceSourceConfig conf) { + SalesforceJobConfiguration jobConf = new SalesforceJobConfiguration(jobConfig); + jobConf.setPluginConfiguration(conf); + + //Depending on conf value fetch the list of fields for each table and create schema object + //return the schema object for each table as SalesforceObjectInfo + List tableInfos = fetchTableInfo(mode, conf); + + jobConf.setTableInfos(tableInfos); + + return tableInfos; + } + + private static List fetchTableInfo(SourceObjectMode mode, SalesforceSourceConfig conf) { + //When mode = Table, fetch details from the table name provided in plugin config + if (mode == SourceObjectMode.SINGLEOBJECT) { + SalesforceObjectInfo tableInfo = getTableMetaData(conf.getObjectName(), conf); + return (tableInfo == null) ? Collections.emptyList() : Collections.singletonList(tableInfo); + } + + //When mode = Reporting, get the list of tables for application name provided in plugin config + //and then fetch details from each of the tables. + List tableInfos = new ArrayList<>(); + + List tableNames = conf.getApplicationName().getTableNames(); + for (String tableName : tableNames) { + SalesforceObjectInfo tableInfo = getTableMetaData(tableName, conf); + if (tableInfo == null) { + continue; + } + tableInfos.add(tableInfo); + } + + return tableInfos; + } + + private static SalesforceObjectInfo getTableMetaData(String tableName, SalesforceSourceConfig conf) { + //Call API to fetch first record from the table + SalesforceTableAPIClientImpl restApi = new SalesforceTableAPIClientImpl(conf); + SalesforceTableDataResponse response = null; + //SalesforceTableDataResponse response = restApi.fetchTableSchema(tableName, + // conf.getStartDate(), conf.getEndDate(), + // true); + if (response == null) { + return null; + } + + List columns = response.getColumns(); + if (columns == null || columns.isEmpty()) { + return null; + } + + SchemaBuilder schemaBuilder = new SchemaBuilder(); + Schema schema = schemaBuilder.constructSchema(tableName, columns); + LOG.debug("table {}, rows = {}", tableName, response.getTotalRecordCount()); + return new SalesforceObjectInfo(tableName, schema, response.getTotalRecordCount()); + } + + @Override + public List getSplits(JobContext jobContext) throws IOException, InterruptedException { + SalesforceJobConfiguration jobConfig = new SalesforceJobConfiguration(jobContext.getConfiguration()); + SalesforceSourceConfig pluginConf = jobConfig.getPluginConf(); + + List tableInfos = jobConfig.getTableInfos(); + List resultSplits = new ArrayList<>(); + + for (SalesforceObjectInfo tableInfo : tableInfos) { + String tableName = tableInfo.getTableName(); + int totalRecords = tableInfo.getRecordCount(); + if (totalRecords < PAGE_SIZE) { + //add single split for table and continue + resultSplits.add(new SalesforceInputSplit(tableName, 0, totalRecords)); + continue; + } + + int pages = (tableInfo.getRecordCount() / PAGE_SIZE) + 1; + int offset = 0; + int recordsOnPage = PAGE_SIZE; + + for (int page = 1; page <= pages; page++) { + if (page == pages) { + recordsOnPage = totalRecords - offset; + } + resultSplits.add(new SalesforceInputSplit(tableName, offset, recordsOnPage)); + offset += PAGE_SIZE; + recordsOnPage -= PAGE_SIZE; + } + } + + return resultSplits; + } + + @Override + public RecordReader createRecordReader(InputSplit inputSplit, + TaskAttemptContext taskAttemptContext) + throws IOException, InterruptedException { + SalesforceJobConfiguration jobConfig = new SalesforceJobConfiguration(taskAttemptContext.getConfiguration()); + SalesforceSourceConfig pluginConf = jobConfig.getPluginConf(); + + return new SalesforceRecordReader(pluginConf); + } +} diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputSplit.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputSplit.java new file mode 100644 index 0000000..2e0c8d7 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputSplit.java @@ -0,0 +1,73 @@ +/* + * Copyright © 2017 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.sfmc.source; + +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.mapreduce.InputSplit; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +/** + * ServiceNow input split + */ +public class SalesforceInputSplit extends InputSplit implements Writable { + private String tableName; + private int offset; + private int length; + + // used by mapreduce + public SalesforceInputSplit() { + } + + public SalesforceInputSplit(String tableName, int offset, int length) { + this.tableName = tableName; + this.offset = offset; + this.length = length; + } + + public String getTableName() { + return tableName; + } + + public int getOffset() { + return offset; + } + + @Override + public void write(DataOutput dataOutput) throws IOException { + dataOutput.writeUTF(this.tableName); + dataOutput.writeInt(this.offset); + } + + @Override + public void readFields(DataInput dataInput) throws IOException { + this.tableName = dataInput.readUTF(); + this.offset = dataInput.readInt(); + } + + @Override + public long getLength() throws IOException, InterruptedException { + return this.length; + } + + @Override + public String[] getLocations() throws IOException, InterruptedException { + return new String[0]; + } +} diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceJobConfiguration.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceJobConfiguration.java new file mode 100644 index 0000000..8e1b3f5 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceJobConfiguration.java @@ -0,0 +1,70 @@ +/* + * Copyright © 2017-2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.sfmc.source; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import io.cdap.plugin.sfmc.source.util.SalesforceObjectInfo; +import org.apache.hadoop.conf.Configuration; + +import java.lang.reflect.Type; +import java.util.List; + +/** + * Allows to specify and access connection configuration properties of {@link Configuration}. + */ +public class SalesforceJobConfiguration { + private static final String PLUGIN_CONF_FIELD = "servicenow.plugin.conf"; + private static final String INFO_FIELD = "servicenow.table.info"; + + private static final Type PLUGIN_CONF_TYPE = new TypeToken() { + }.getType(); + private static final Type INFO_TYPE = new TypeToken>() { + }.getType(); + + private static final Gson GSON = new Gson(); + + private Configuration conf; + + public SalesforceJobConfiguration(Configuration job) { + this.conf = job; + } + + public Configuration getConf() { + return this.conf; + } + + public void setPluginConfiguration(SalesforceSourceConfig conf) { + set(PLUGIN_CONF_FIELD, GSON.toJson(conf)); + } + + public SalesforceSourceConfig getPluginConf() { + return GSON.fromJson(getConf().get(PLUGIN_CONF_FIELD), PLUGIN_CONF_TYPE); + } + + public List getTableInfos() { + return GSON.fromJson(getConf().get(INFO_FIELD), INFO_TYPE); + } + + public void setTableInfos(List infoList) { + set(INFO_FIELD, GSON.toJson(infoList)); + } + + private void set(String key, String value) { + getConf().set(key, value); + } +} diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java new file mode 100644 index 0000000..dcbd5f3 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java @@ -0,0 +1,271 @@ +/* + * Copyright © 2017-2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.sfmc.source; + +import com.google.common.base.Strings; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; + +import io.cdap.plugin.sfmc.source.apiclient.SalesforceTableAPIClientImpl; +import io.cdap.plugin.sfmc.source.apiclient.SalesforceTableDataResponse; +import io.cdap.plugin.sfmc.source.util.SchemaBuilder; +import io.cdap.plugin.sfmc.source.util.SourceObjectMode; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.sql.SQLException; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PAGE_SIZE; + +/** + * Record reader that reads the entire contents of a ServiceNow table. + */ +public class SalesforceRecordReader extends RecordReader { + private static final Logger LOG = LoggerFactory.getLogger(SalesforceRecordReader.class); + private final SalesforceSourceConfig pluginConf; + private SalesforceInputSplit split; + private int pos; + private List tableFields; + private Schema schema; + + private String tableName; + + private List> results; + private Iterator> iterator; + private Map row; + + SalesforceRecordReader(SalesforceSourceConfig pluginConf) { + this.pluginConf = pluginConf; + } + + @Override + public void initialize(InputSplit split, TaskAttemptContext context) { + this.split = (SalesforceInputSplit) split; + this.pos = 0; + } + + @Override + public boolean nextKeyValue() throws IOException { + try { + if (results == null) { + fetchData(); + } + + if (!iterator.hasNext()) { + return false; + } + + row = iterator.next(); + + pos++; + } catch (Exception e) { + LOG.error("Error in nextKeyValue", e); + throw new IOException("Exception in nextKeyValue", e); + } + return true; + } + + @Override + public NullWritable getCurrentKey() { + return NullWritable.get(); + } + + @Override + public StructuredRecord getCurrentValue() throws IOException { + StructuredRecord.Builder recordBuilder = StructuredRecord.builder(schema); + + if (pluginConf.getQueryMode() == SourceObjectMode.MULTIOBJECT) { + + } + + try { + for (Schema.Field field : tableFields) { + String fieldName = field.getName(); + Object fieldValue = convertToValue(fieldName, field.getSchema(), row); + recordBuilder.set(fieldName, fieldValue); + } + } catch (Exception e) { + LOG.error("Error decoding row from table " + tableName, e); + throw new IOException("Error decoding row from table " + tableName, e); + } + return recordBuilder.build(); + } + + @Override + public float getProgress() throws IOException, InterruptedException { + return pos / (float) split.getLength(); + } + + @Override + public void close() throws IOException { + SQLException exception = null; + + if (exception != null) { + throw new IOException(exception); + } + } + + private void fetchData() { + tableName = split.getTableName(); + + + SalesforceTableAPIClientImpl restApi = new SalesforceTableAPIClientImpl(pluginConf); + + //Get the table data + // results = restApi.fetchTableRecords(tableName, pluginConf.getStartDate(), pluginConf.getEndDate(), + // split.getOffset(), PAGE_SIZE); + + LOG.debug("size={}", results.size()); + if (!results.isEmpty()) { + fetchSchema(restApi); + } + + iterator = results.iterator(); + } + + private void fetchSchema(SalesforceTableAPIClientImpl restApi) { + //Fetch the column definition + // SalesforceTableDataResponse response = restApi.fetchTableSchema(tableName, null, null, + // false); + SalesforceTableDataResponse response = null; + if (response == null) { + return; + } + + //Build schema + SchemaBuilder schemaBuilder = new SchemaBuilder(); + Schema tempSchema = schemaBuilder.constructSchema(tableName, response.getColumns()); + tableFields = tempSchema.getFields(); + List schemaFields = new ArrayList<>(tableFields); + + if (pluginConf.getQueryMode() == SourceObjectMode.MULTIOBJECT) { + + } + + schema = Schema.recordOf(tableName, schemaFields); + } + + private Object convertToValue(String fieldName, Schema fieldSchema, Map record) { + Schema.Type fieldType = fieldSchema.getType(); + Object fieldValue = record.get(fieldName); + + switch (fieldType) { + case STRING: + return convertToStringValue(fieldValue); + case DOUBLE: + return convertToDoubleValue(fieldValue); + case INT: + return convertToIntegerValue(fieldValue); + case BOOLEAN: + return convertToBooleanValue(fieldValue); + //case RECORD: + //return convertToRecordValue(fieldName, fieldSchema, fieldValue); + /* + if (fieldValue instanceof Map) { + Map nestedRecord = (Map) fieldValue; + StructuredRecord.Builder nestedRecordBuilder = StructuredRecord.builder(fieldSchema); + Objects.requireNonNull(fieldSchema.getFields(), "Nested Schema fields cannot be empty").forEach( + nestedField -> { + String nestedFieldName = nestedField.getName(); + Object nestedFieldValue = convertToValue(nestedFieldName, nestedField.getSchema(), nestedRecord); + nestedRecordBuilder.set(nestedFieldName, nestedFieldValue); + } + ); + return nestedRecordBuilder.build(); + } else if (fieldValue instanceof String && Strings.isNullOrEmpty(String.valueOf(fieldValue))) { + return null; + } else { + throw new IllegalStateException( + String.format("Field '%s' is of unexpected type '%s'. Declared 'RECORD' types: %s", + fieldName, record.get(fieldName).getClass().getSimpleName(), fieldSchema.toString())); + } + */ + case UNION: + if (fieldSchema.isNullable()) { + return convertToValue(fieldName, fieldSchema.getNonNullable(), record); + } + throw new IllegalStateException( + String.format("Field '%s' is of unexpected type '%s'. Declared 'complex UNION' types: %s", + fieldName, record.get(fieldName).getClass().getSimpleName(), fieldSchema.getUnionSchemas())); + default: + throw new IllegalStateException( + String.format("Record type '%s' is not supported for field '%s'", fieldType.name(), fieldName)); + } + } + + private String convertToStringValue(Object fieldValue) { + return String.valueOf(fieldValue); + } + + private Double convertToDoubleValue(Object fieldValue) { + if (fieldValue instanceof String && Strings.isNullOrEmpty(String.valueOf(fieldValue))) { + return null; + } + + return Double.parseDouble(String.valueOf(fieldValue)); + } + + private Integer convertToIntegerValue(Object fieldValue) { + if (fieldValue instanceof String && Strings.isNullOrEmpty(String.valueOf(fieldValue))) { + return null; + } + + return Integer.parseInt(String.valueOf(fieldValue)); + } + + private Boolean convertToBooleanValue(Object fieldValue) { + if (fieldValue instanceof String && Strings.isNullOrEmpty(String.valueOf(fieldValue))) { + return null; + } + + return Boolean.parseBoolean(String.valueOf(fieldValue)); + } + + private StructuredRecord convertToRecordValue(String fieldName, Schema fieldSchema, Object fieldValue) { + if (fieldValue instanceof String && Strings.isNullOrEmpty(String.valueOf(fieldValue))) { + return null; + } + + if (fieldValue instanceof Map) { + Map nestedRecord = (Map) fieldValue; + StructuredRecord.Builder nestedRecordBuilder = StructuredRecord.builder(fieldSchema); + Objects.requireNonNull(fieldSchema.getFields(), "Nested Schema fields cannot be empty").forEach( + nestedField -> { + String nestedFieldName = nestedField.getName(); + Object nestedFieldValue = convertToValue(nestedFieldName, nestedField.getSchema(), nestedRecord); + nestedRecordBuilder.set(nestedFieldName, nestedFieldValue); + } + ); + return nestedRecordBuilder.build(); + } else { + throw new IllegalStateException( + String.format("Field '%s' is of unexpected type '%s'. Declared 'RECORD' types: %s", + fieldName, fieldValue.getClass().getSimpleName(), fieldSchema.toString())); + } + } +} diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java new file mode 100644 index 0000000..bab926e --- /dev/null +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java @@ -0,0 +1,101 @@ +/* + * Copyright © 2017-2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.sfmc.source; + +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.api.annotation.Plugin; +import io.cdap.cdap.api.data.batch.Input; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.dataset.lib.KeyValue; +import io.cdap.cdap.etl.api.Emitter; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.cdap.etl.api.PipelineConfigurer; +import io.cdap.cdap.etl.api.StageConfigurer; +import io.cdap.cdap.etl.api.action.SettableArguments; +import io.cdap.cdap.etl.api.batch.BatchSource; +import io.cdap.cdap.etl.api.batch.BatchSourceContext; +import io.cdap.plugin.common.SourceInputFormatProvider; +//import io.cdap.plugin.servicenow.source.util.SalesforceObjectInfo; +//import io.cdap.plugin.servicenow.source.util.SourceObjectMode; +import io.cdap.plugin.sfmc.source.util.SalesforceObjectInfo; +import io.cdap.plugin.sfmc.source.util.SourceObjectMode; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.NullWritable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; + +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PLUGIN_NAME; + + +/** + * A {@link BatchSource} that reads data from multiple tables in Service Now. + */ +@Plugin(type = BatchSource.PLUGIN_TYPE) +@Name(PLUGIN_NAME) +@Description("Reads from multiple tables in Service Now. " + + "Outputs one record for each row in each table, with the table name as a record field. " + + "Also sets a pipeline argument for each table read, which contains the table schema. ") +public class SalesforceSource extends BatchSource { + private static final Logger LOG = LoggerFactory.getLogger(SalesforceSource.class); + + private final SalesforceSourceConfig conf; + + public SalesforceSource(SalesforceSourceConfig conf) { + this.conf = conf; + } + + @Override + public void configurePipeline(PipelineConfigurer pipelineConfigurer) { + super.configurePipeline(pipelineConfigurer); + + LOG.debug("Validate config during `configurePipeline` stage: {}", conf); + StageConfigurer stageConfigurer = pipelineConfigurer.getStageConfigurer(); + FailureCollector collector = stageConfigurer.getFailureCollector(); + + conf.validate(collector); + // Since we have validated all the properties, throw an exception if there are any errors in the collector. + // This is to avoid adding same validation errors again in getSchema method call + collector.getOrThrowException(); + } + + @Override + public void prepareRun(BatchSourceContext context) throws Exception { + FailureCollector collector = context.getFailureCollector(); + conf.validate(collector); + collector.getOrThrowException(); + + SourceObjectMode mode = conf.getQueryMode(collector); + + Configuration hConf = new Configuration(); + Collection tables = SalesforceInputFormat.setInput(hConf, mode, conf); + /* SettableArguments arguments = context.getArguments(); + for (SalesforceObjectInfo tableInfo : tables) { + arguments.set(TABLE_PREFIX + tableInfo.getTableName(), tableInfo.getSchema().toString()); + } + + context.setInput(Input.of(conf.getReferenceName(), + new SourceInputFormatProvider(ServiceNowInputFormat.class, hConf)));*/ + } + + @Override + public void transform(KeyValue input, Emitter emitter) { + emitter.emit(input.getValue()); + } +} diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java new file mode 100644 index 0000000..7823ceb --- /dev/null +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java @@ -0,0 +1,390 @@ +/* + * Copyright © 2017-2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.sfmc.source; + +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Macro; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.api.plugin.PluginConfig; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.plugin.common.Constants; +import io.cdap.plugin.sfmc.source.apiclient.SalesforceTableAPIClientImpl; +import io.cdap.plugin.sfmc.source.util.SourceApplication; +import io.cdap.plugin.sfmc.source.util.SourceObjectMode; +import io.cdap.plugin.sfmc.source.util.SourceValueType; +import io.cdap.plugin.sfmc.source.util.Util; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +import java.time.LocalDate; +import java.time.temporal.ChronoUnit; +import java.util.Optional; +import javax.annotation.Nullable; + +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.DATE_FORMAT; +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_API_ENDPOINT; +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_AUTH_API_ENDPOINT; +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_CLIENT_ID; +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_CLIENT_SECRET; +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_END_DATE; +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_MULTIOBJECT_NAME; +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_OBJECT_NAME; +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_QUERY_MODE; +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_SOAP_API_ENDPOINT; +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_START_DATE; + + + +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_VALUE_TYPE; +/** + * Configuration for the {@link SalesforceSource}. + */ +public class SalesforceSourceConfig extends PluginConfig { + private static final Logger LOG = LoggerFactory.getLogger(SalesforceSourceConfig.class); + @Name(Constants.Reference.REFERENCE_NAME) + @Description("This will be used to uniquely identify this source for lineage, annotating metadata, etc.") + private String referenceName; + + @Name(PROPERTY_QUERY_MODE) + @Macro + @Description("Mode of query. The mode can be one of two values: " + + "`Reporting` - will allow user to choose application for which data will be fetched for all tables, " + + "`Table` - will allow user to enter table name for which data will be fetched.") + private String queryMode; + + @Name(PROPERTY_MULTIOBJECT_NAME) + @Macro + @Nullable + @Description("Application name for which data to be fetched. The application can be one of three values: " + + "`Contract Management` - will fetch data for all tables under Contract Management application, " + + "`Product Catalog` - will fetch data for all tables under Product Catalog application, " + + "`Procurement` - will fetch data for all tables under Procurement application. " + + "Note, the Application name value will be ignored if the Mode is set to `Table`.") + private String multiObjectName; + + @Name(PROPERTY_OBJECT_NAME) + @Macro + @Nullable + @Description("The name of the Salesforce table from which data to be fetched. Note, the Table name value " + + "will be ignored if the Mode is set to `Reporting`.") + private String objectName; + + @Name(PROPERTY_CLIENT_ID) + @Macro + @Description(" The Client ID for Salesforce Instance.") + private String clientId; + + @Name(PROPERTY_CLIENT_SECRET) + @Macro + @Description("The Client Secret for Salesforce Instance.") + private String clientSecret; + + @Name(PROPERTY_API_ENDPOINT) + @Macro + @Description("The REST API Endpoint for Salesforce Instance. For example, https://instance.service-now.com") + private String restEndpoint; + + @Name(PROPERTY_AUTH_API_ENDPOINT) + @Macro + @Description("The AUTH API Endpoint for Salesforce Instance. " + + "For example, https://instance.auth.marketingcloudapis.com") + private String authEndpoint; + + @Name(PROPERTY_SOAP_API_ENDPOINT) + @Macro + @Description("The SOAP API Endpoint for Salesforce Instance. " + + "For example https://instance.Salesforce.soap.marketingcloudapis.com/Service.asmx") + private String soapEndpoint; + + @Name(PROPERTY_VALUE_TYPE) + @Macro + @Nullable + @Description("The type of values to be returned. The type can be one of two values: " + + "`Actual` - will fetch the actual values from the Salesforce tables, " + + "`Display` - will fetch the display values from the Salesforce tables.") + private String valueType; + + @Name(PROPERTY_START_DATE) + @Macro + @Nullable + @Description("The Start date to be used to filter the data. The format must be 'yyyy-MM-dd'.") + private String startDate; + + @Name(PROPERTY_END_DATE) + @Macro + @Nullable + @Description("The End date to be used to filter the data. The format must be 'yyyy-MM-dd'.") + private String endDate; + + public SalesforceSourceConfig(String referenceName, String queryMode, @Nullable String multiObjectName, + @Nullable String objectName, String clientId, + String clientSecret, String restEndpoint, String authEndpoint, String soapEndpoint, + @Nullable String valueType, @Nullable String startDate, @Nullable String endDate) { + this.referenceName = referenceName; + this.queryMode = queryMode; + this.multiObjectName = multiObjectName; + + this.objectName = objectName; + this.clientId = clientId; + this.clientSecret = clientSecret; + this.restEndpoint = restEndpoint; + this.authEndpoint = authEndpoint; + this.soapEndpoint = soapEndpoint; + this.valueType = valueType; + this.startDate = startDate; + this.endDate = endDate; + } + + public String getReferenceName() { + return referenceName; + } + + public SourceObjectMode getQueryMode(FailureCollector collector) { + SourceObjectMode mode = getQueryMode(); + if (mode != null) { + return mode; + } + + collector.addFailure("Unsupported query mode value: " + queryMode, + String.format("Supported modes are: %s", SourceObjectMode.getSupportedModes())) + .withConfigProperty(PROPERTY_QUERY_MODE); + throw collector.getOrThrowException(); + } + + public SourceObjectMode getQueryMode() { + Optional sourceQueryMode = SourceObjectMode.fromValue(queryMode); + + return sourceQueryMode.isPresent() ? sourceQueryMode.get() : null; + } + + public SourceApplication getApplicationName(FailureCollector collector) { + SourceApplication application = getApplicationName(); + if (application != null) { + return application; + } + + collector.addFailure("Unsupported multiObject name value: " + multiObjectName, + String.format("Supported applications are: %s", SourceApplication.getSupportedApplications())) + .withConfigProperty(PROPERTY_MULTIOBJECT_NAME); + throw collector.getOrThrowException(); + } + + @Nullable + public SourceApplication getApplicationName() { + Optional sourceApplication = SourceApplication.fromValue(multiObjectName); + + return sourceApplication.isPresent() ? sourceApplication.get() : null; + } + + + + @Nullable + public String getObjectName() { + return objectName; + } + + public String getClientId() { + return clientId; + } + + public String getClientSecret() { + return clientSecret; + } + + public String getRestEndpoint() { + return restEndpoint; + } + + public String getAuthEndpoint() { + return authEndpoint; + } + + public String getSoapEndpoint() { + return soapEndpoint; + } + + public SourceValueType getValueType(FailureCollector collector) { + SourceValueType type = getValueType(); + if (type != null) { + return type; + } + + collector.addFailure("Unsupported type value: " + valueType, + String.format("Supported value types are: %s", SourceValueType.getSupportedValueTypes())) + .withConfigProperty(PROPERTY_VALUE_TYPE); + throw collector.getOrThrowException(); + } + + @Nullable + public SourceValueType getValueType() { + Optional sourceValueType = SourceValueType.fromValue(valueType); + + return sourceValueType.isPresent() ? sourceValueType.get() : null; + } + + @Nullable + public String getStartDate() { + return startDate; + } + + @Nullable + public String getEndDate() { + return endDate; + } + + /** + * Validates {@link SalesforceSourceConfig} instance. + */ + public void validate(FailureCollector collector) { + //Validates the given referenceName to consists of characters allowed to represent a dataset. + //IdUtils.validateReferenceName(referenceName, collector); + + validateCredentials(collector); + validateQueryMode(collector); + getValueType(collector); + validateDateRange(collector); + } + + private void validateCredentials(FailureCollector collector) { + if (!shouldConnect()) { + return; + } + + if (Util.isNullOrEmpty(clientId)) { + collector.addFailure("Client ID must be specified.", null) + .withConfigProperty(PROPERTY_CLIENT_ID); + } + + if (Util.isNullOrEmpty(clientSecret)) { + collector.addFailure("Client Secret must be specified.", null) + .withConfigProperty(PROPERTY_CLIENT_SECRET); + } + + if (Util.isNullOrEmpty(restEndpoint)) { + collector.addFailure(" REST Endpoint must be specified.", null) + .withConfigProperty(PROPERTY_API_ENDPOINT); + } + + if (Util.isNullOrEmpty(authEndpoint)) { + collector.addFailure("Auth Endpoint must be specified.", null) + .withConfigProperty(PROPERTY_AUTH_API_ENDPOINT); + } + + if (Util.isNullOrEmpty(soapEndpoint)) { + collector.addFailure("Soap Endpoint must be specified.", null) + .withConfigProperty(PROPERTY_SOAP_API_ENDPOINT); + } + + try { + LOG.info("Called SalesforceTableAPIClientImpl"); + SalesforceTableAPIClientImpl restApi = new SalesforceTableAPIClientImpl(this); + LOG.info("Completed SalesforceTableAPIClientImpl"); + // restApi.getAccessToken(); + //1- TODO + } catch (Exception e) { + collector.addFailure("Unable to connect to Salesforce Instance.", + "Ensure properties like Client ID, Client Secret, API Endpoint, Soap Endpoint, Auth Endpoint " + + "are correct.") + .withConfigProperty(PROPERTY_CLIENT_ID) + .withConfigProperty(PROPERTY_CLIENT_SECRET) + .withConfigProperty(PROPERTY_API_ENDPOINT) + .withConfigProperty(PROPERTY_AUTH_API_ENDPOINT) + .withConfigProperty(PROPERTY_SOAP_API_ENDPOINT) + .withStacktrace(e.getStackTrace()); + } + } + + private void validateQueryMode(FailureCollector collector) { + //according to query mode check if either table name/application exists or not + if (containsMacro(PROPERTY_QUERY_MODE)) { + return; + } + + SourceObjectMode mode = getQueryMode(collector); + + if (mode == SourceObjectMode.MULTIOBJECT) { + validateMultiObjectQueryMode(collector); + } else { + validateObjectQueryMode(collector); + } + } + + private void validateMultiObjectQueryMode(FailureCollector collector) { + if (!containsMacro(PROPERTY_MULTIOBJECT_NAME)) { + getApplicationName(collector); + } + + + + + } + + private void validateObjectQueryMode(FailureCollector collector) { + if (containsMacro(PROPERTY_OBJECT_NAME)) { + return; + } + + if (Util.isNullOrEmpty(objectName)) { + collector.addFailure("Object name must be specified.", null) + .withConfigProperty(PROPERTY_OBJECT_NAME); + } + } + + private void validateDateRange(FailureCollector collector) { + if (containsMacro(PROPERTY_START_DATE) || containsMacro(PROPERTY_END_DATE)) { + return; + } + + if (Util.isNullOrEmpty(startDate) || Util.isNullOrEmpty(endDate)) { + return; + } + + //validate the date formats for both start date & end date + if (!Util.isValidDateFormat(DATE_FORMAT, startDate)) { + collector.addFailure("Invalid format for Start date. Correct Format: " + DATE_FORMAT, null) + .withConfigProperty(PROPERTY_START_DATE); + } + + if (!Util.isValidDateFormat(DATE_FORMAT, endDate)) { + collector.addFailure("Invalid format for End date. Correct Format:" + DATE_FORMAT, null) + .withConfigProperty(PROPERTY_END_DATE); + } + + //validate the date range by checking if start date is smaller than end date + LocalDate fromDate = LocalDate.parse(startDate); + LocalDate toDate = LocalDate.parse(endDate); + long noOfDays = ChronoUnit.DAYS.between(fromDate, toDate); + + if (noOfDays < 0) { + collector.addFailure("End date must be greater than Start date.", null) + .withConfigProperty(PROPERTY_START_DATE) + .withConfigProperty(PROPERTY_END_DATE); + } + } + + /** + * Returns true if Salesforce can be connected to. + */ + public boolean shouldConnect() { + return !containsMacro(PROPERTY_CLIENT_ID) && + !containsMacro(PROPERTY_CLIENT_SECRET) && + !containsMacro(PROPERTY_API_ENDPOINT) && + !containsMacro(PROPERTY_AUTH_API_ENDPOINT) && + !containsMacro(PROPERTY_SOAP_API_ENDPOINT); + } +} diff --git a/src/main/java/io/cdap/plugin/sfmc/source/apiclient/SalesforceTableAPIClientImpl.java b/src/main/java/io/cdap/plugin/sfmc/source/apiclient/SalesforceTableAPIClientImpl.java new file mode 100644 index 0000000..d80ef31 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sfmc/source/apiclient/SalesforceTableAPIClientImpl.java @@ -0,0 +1,93 @@ +/* + * Copyright © 2017-2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.sfmc.source.apiclient; + +import com.exacttarget.fuelsdk.ETClient; +import com.exacttarget.fuelsdk.ETSdkException; +import com.google.common.base.Strings; +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.reflect.TypeToken; + +import io.cdap.plugin.sfmc.source.SalesforceSourceConfig; +import io.cdap.plugin.sfmc.source.util.SalesforceColumn; +import io.cdap.plugin.sfmc.source.util.TokenGenerator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * Implementation class for ServiceNow Table API + */ +public class SalesforceTableAPIClientImpl { + private static final Logger LOG = LoggerFactory.getLogger(SalesforceTableAPIClientImpl.class); + + + private SalesforceSourceConfig conf; + + public SalesforceTableAPIClientImpl(SalesforceSourceConfig conf) { + this.conf = conf; + + TokenGenerator tokenGenerator = new TokenGenerator(); + try { + ETClient client = tokenGenerator.getAPIDetails(conf); + client.getAccessToken(); + LOG.info("token{}, client{}", client.getAccessToken(), client.getClientId()); + } catch (Exception ets) { + ets.printStackTrace(); + } + } + + + private List> parseResponseToResultListOfMap(String responseBody) { + Gson gson = new Gson(); + JsonObject jo = gson.fromJson(responseBody, JsonObject.class); + JsonArray ja = jo.getAsJsonArray("result"); + + Type type = new TypeToken>>() { + }.getType(); + + return gson.fromJson(ja, type); + } + + private List parseResponseToResultListOfPojo(String responseBody) { + Gson gson = new Gson(); + JsonObject jo = gson.fromJson(responseBody, JsonObject.class); + JsonArray ja = jo.getAsJsonArray("result"); + + Type type = new TypeToken>() { + }.getType(); + + return gson.fromJson(ja, type); + } + + private String getErrorMessage(String responseBody) { + try { + Gson gson = new Gson(); + JsonObject jo = gson.fromJson(responseBody, JsonObject.class); + return jo.getAsJsonObject("error").get("message").getAsString(); + } catch (Exception e) { + return e.getMessage(); + } + } +} diff --git a/src/main/java/io/cdap/plugin/sfmc/source/apiclient/SalesforceTableDataResponse.java b/src/main/java/io/cdap/plugin/sfmc/source/apiclient/SalesforceTableDataResponse.java new file mode 100644 index 0000000..afa159a --- /dev/null +++ b/src/main/java/io/cdap/plugin/sfmc/source/apiclient/SalesforceTableDataResponse.java @@ -0,0 +1,59 @@ +/* + * Copyright © 2017-2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.sfmc.source.apiclient; + + + +import io.cdap.plugin.sfmc.source.util.SalesforceColumn; + +import java.util.List; +import java.util.Map; + +/** + * A Pojo class to wrap the success response for ServiceNow Table data + */ +public class SalesforceTableDataResponse { + private int totalRecordCount; + + private List columns; + + private List> result; + + public int getTotalRecordCount() { + return totalRecordCount; + } + + public void setTotalRecordCount(int totalRecordCount) { + this.totalRecordCount = totalRecordCount; + } + + public List getColumns() { + return columns; + } + + public void setColumns(List columns) { + this.columns = columns; + } + + public List> getResult() { + return result; + } + + public void setResult(List> result) { + this.result = result; + } +} diff --git a/src/main/java/io/cdap/plugin/sfmc/source/apiclient/ServiceNowTableAPIRequestBuilder.java b/src/main/java/io/cdap/plugin/sfmc/source/apiclient/ServiceNowTableAPIRequestBuilder.java new file mode 100644 index 0000000..013de61 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sfmc/source/apiclient/ServiceNowTableAPIRequestBuilder.java @@ -0,0 +1,75 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.sfmc.source.apiclient; + +import com.google.common.base.Joiner; + +import io.cdap.plugin.sfmc.source.util.SourceValueType; + +import java.net.URLEncoder; +import java.util.Arrays; + +/** + * ServiceNowTableAPIRequestBuilder + */ +public class ServiceNowTableAPIRequestBuilder { + private static final String TABLE_API_URL_TEMPLATE = "%s/api/now/table/%s"; +/* + public ServiceNowTableAPIRequestBuilder(String instanceBaseUrl, String tableName) { + super(String.format(TABLE_API_URL_TEMPLATE, instanceBaseUrl, tableName)); + } + + public ServiceNowTableAPIRequestBuilder setQuery(String query) { + try { + this.parameters.put("sysparm_query", URLEncoder.encode(query, "UTF-8")); + } catch (Exception e) { + } + return this; + } + + public ServiceNowTableAPIRequestBuilder setOffset(int offset) { + this.parameters.put("sysparm_offset", String.valueOf(offset)); + return this; + } + + public ServiceNowTableAPIRequestBuilder setLimit(int limit) { + this.parameters.put("sysparm_limit", String.valueOf(limit)); + return this; + } + + public ServiceNowTableAPIRequestBuilder setFields(String... fields) { + if (fields == null || fields.length == 0) { + return this; + } + + try { + this.parameters.put("sysparm_fields", URLEncoder.encode(Joiner.on(',').join(Arrays.asList(fields)), "UTF-8")); + } catch (Exception e) { + } + return this; + } + + public ServiceNowTableAPIRequestBuilder setDisplayValue(SourceValueType displayValue) { + this.parameters.put("sysparm_display_value", displayValue.getValue()); + return this; + } + + public ServiceNowTableAPIRequestBuilder setExcludeReferenceLink(boolean excludeRefLink) { + this.parameters.put("sysparm_exclude_reference_link", String.valueOf(excludeRefLink)); + return this; + }*/ +} diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceColumn.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceColumn.java new file mode 100644 index 0000000..23a31a1 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceColumn.java @@ -0,0 +1,54 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.sfmc.source.util; + +import com.google.common.base.Strings; +import com.google.gson.annotations.SerializedName; + +/** + * Information about a ServiceNow table column. + */ +public class SalesforceColumn { + @SerializedName("element") + private String fieldName; + + @SerializedName("internal_type") + private String typeName; + + private String mandatory; + + public SalesforceColumn() { + } + + public SalesforceColumn(String fieldName, String typeName) { + this.fieldName = fieldName; + this.typeName = typeName; + this.mandatory = "false"; + } + + public String getFieldName() { + return fieldName; + } + + public String getTypeName() { + return typeName; + } + + public boolean isMandatory() { + return !Strings.isNullOrEmpty(mandatory) && "true".equalsIgnoreCase(mandatory); + } +} diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java new file mode 100644 index 0000000..a9be2cf --- /dev/null +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java @@ -0,0 +1,93 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.sfmc.source.util; + +/** + * ServiceNow constants. + */ +public interface SalesforceConstants { + + /** + * ServiceNow plugin name. + */ + String PLUGIN_NAME = "Salesforce"; + + /** + * Configuration property name used to specify the query mode. + */ + String PROPERTY_QUERY_MODE = "queryMode"; + + /** + * Configuration property name used to specify the MultiObject. + */ + String PROPERTY_MULTIOBJECT_NAME = "multiObjectName"; + + /** + * Configuration property name used to specify the table name field. + */ + String PROPERTY_TABLE_NAME_FIELD = "tableNameField"; + + /** + * Configuration property name used to specify Object name + */ + String PROPERTY_OBJECT_NAME = "objectName"; + + /** + * Configuration property name used to specify client id + */ + String PROPERTY_CLIENT_ID = "clientId"; + + /** + * Configuration property name used to specify client secret + */ + String PROPERTY_CLIENT_SECRET = "clientSecret"; + + /** + * Configuration property name used to specify REST API endpoint + */ + String PROPERTY_API_ENDPOINT = "restEndpoint"; + + /** + * Configuration property name used to specify user name + */ + String PROPERTY_AUTH_API_ENDPOINT = "authEndpoint"; + + /** + * Configuration property name used to specify password + */ + String PROPERTY_SOAP_API_ENDPOINT = "soapEndpoint"; + + /** + * Configuration property name used to specify value type + */ + String PROPERTY_VALUE_TYPE = "valueType"; + + /** + * Configuration property name used to specify start date + */ + String PROPERTY_START_DATE = "startDate"; + + /** + * Configuration property name used to specify end date + */ + String PROPERTY_END_DATE = "endDate"; + + String TABLE_PREFIX = "multisink."; + String DATE_FORMAT = "yyyy-MM-dd"; + + int PAGE_SIZE = 5000; +} diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceObjectInfo.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceObjectInfo.java new file mode 100644 index 0000000..64b8ab8 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceObjectInfo.java @@ -0,0 +1,46 @@ +/* + * Copyright © 2017-2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.sfmc.source.util; + +import io.cdap.cdap.api.data.schema.Schema; + +/** + * Information about a ServiceNow table. + */ +public class SalesforceObjectInfo { + private final String tableName; + private final Schema schema; + private final int recordCount; + + public SalesforceObjectInfo(String tableName, Schema schema, int recordCount) { + this.tableName = tableName; + this.schema = schema; + this.recordCount = recordCount; + } + + public String getTableName() { + return tableName; + } + + public Schema getSchema() { + return schema; + } + + public int getRecordCount() { + return recordCount; + } +} diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SchemaBuilder.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SchemaBuilder.java new file mode 100644 index 0000000..3c15486 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SchemaBuilder.java @@ -0,0 +1,90 @@ +/* + * Copyright © 2017-2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.sfmc.source.util; + +import com.google.common.base.Strings; +import io.cdap.cdap.api.data.schema.Schema; + +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * Utility class to build schema + */ +public class SchemaBuilder { + public Schema constructSchema(String tableName, List columns) { + SchemaBuilder schemaBuilder = new SchemaBuilder(); + List fields = schemaBuilder.constructSchemaFields(columns); + + return Schema.recordOf(tableName, fields); + } + + private List constructSchemaFields(List columns) { + return columns.stream() + .map(o -> transformToField(o)) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + private Schema.Field transformToField(SalesforceColumn column) { + String name = column.getFieldName(); + if (Strings.isNullOrEmpty(name)) { + return null; + } + + Schema schema = createSchema(column); + if (schema == null) { + return null; + } + + return Schema.Type.NULL == schema.getType() + ? Schema.Field.of(name, schema) + : Schema.Field.of(name, Schema.nullableOf(schema)); + } + + private Schema createSchema(SalesforceColumn column) { + switch (column.getTypeName().toLowerCase()) { + case "decimal": + return Schema.of(Schema.Type.DOUBLE); + case "integer": + return Schema.of(Schema.Type.INT); + case "boolean": + return Schema.of(Schema.Type.BOOLEAN); + case "reference": + /* + List fields = Arrays.asList( + Schema.Field.of("link", Schema.nullableOf(Schema.of(Schema.Type.STRING))), + Schema.Field.of("value", Schema.nullableOf(Schema.of(Schema.Type.STRING))) + ); + return Schema.recordOf(column.getFieldName(), fields); + */ + case "currency": + case "glide_date": + case "glide_date_time": + case "sys_class_name": + case "domain_id": + case "domain_path": + case "guid": + case "translated_html": + case "journal": + case "string": + default: + return Schema.of(Schema.Type.STRING); + } + } +} diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SourceApplication.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SourceApplication.java new file mode 100644 index 0000000..d8dfa24 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SourceApplication.java @@ -0,0 +1,80 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.sfmc.source.util; + +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Indicates application for which data to be fetched from multiple tables in ServiceNow. + */ +public enum SourceApplication { + + /** + * Data from tables related to Contract Management will be fetched + */ + CONTRACT_MANAGEMENT("Contract Management", Arrays.asList("clm_m2m_contract_asset", "clm_m2m_rate_card_asset", + "clm_condition_checker", "clm_condition_check", "ast_contract", "clm_contract_history", + "clm_terms_and_conditions", "clm_m2m_contract_and_terms", "clm_m2m_contract_user")), + + /** + * Data from tables related to Product Catalog will be fetched + */ + PRODUCT_CATALOG("Product Catalog", Arrays.asList("new_call", "pc_hardware_cat_item", "pc_product_cat_item", + "pc_software_cat_item", "pc_vendor_cat_item")), + + /** + * Data from tables related to Procurement will be fetched + */ + PROCUREMENT("Procurement", Arrays.asList("proc_po", "proc_po_item", "proc_rec_slip", "proc_rec_slip_item")); + + private final String value; + private final List tableNames; + + SourceApplication(String value, List tableNames) { + this.value = value; + this.tableNames = tableNames; + } + + /** + * Converts application string value into {@link SourceApplication} enum. + * + * @param stringValue application string value + * @return source application in optional container + */ + public static Optional fromValue(String stringValue) { + return Stream.of(values()) + .filter(keyType -> keyType.value.equalsIgnoreCase(stringValue)) + .findAny(); + } + + public static String getSupportedApplications() { + return Arrays.stream(SourceApplication.values()).map(SourceApplication::getValue) + .collect(Collectors.joining(", ")); + } + + public String getValue() { + return value; + } + + public List getTableNames() { + return tableNames; + } +} diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SourceObjectMode.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SourceObjectMode.java new file mode 100644 index 0000000..92ac468 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SourceObjectMode.java @@ -0,0 +1,65 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.sfmc.source.util; + +import java.util.Arrays; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Indicates query mode which will be used when fetching ServiceNow data. + */ +public enum SourceObjectMode { + + /** + * Mode will be used as Reporting for querying data + */ + MULTIOBJECT("Multi-Object"), + + /** + * Mode will be used as Table for querying data + */ + SINGLEOBJECT("Single-Object"); + + private final String value; + + SourceObjectMode(String value) { + this.value = value; + } + + /** + * Converts mode string value into {@link SourceObjectMode} enum. + * + * @param stringValue mode string value + * @return source query mode in optional container + */ + public static Optional fromValue(String stringValue) { + return Stream.of(values()) + .filter(keyType -> keyType.value.equalsIgnoreCase(stringValue)) + .findAny(); + } + + public static String getSupportedModes() { + return Arrays.stream(SourceObjectMode.values()).map(SourceObjectMode::getValue) + .collect(Collectors.joining(", ")); + } + + public String getValue() { + return value; + } +} diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SourceValueType.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SourceValueType.java new file mode 100644 index 0000000..65fb1e2 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SourceValueType.java @@ -0,0 +1,71 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.sfmc.source.util; + +import java.util.Arrays; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Indicates type of value to be returned from ServiceNow Table API + */ +public enum SourceValueType { + + /** + * Actual values will be returned by ServiceNow Table API + */ + SHOW_ACTUAL_VALUE("Actual", "false"), + + /** + * Display values will be returned by ServiceNow Table API + */ + SHOW_DISPLAY_VALUE("Display", "true"); + + private final String valueType; + private final String value; + + SourceValueType(String valueType, String value) { + this.valueType = valueType; + this.value = value; + } + + /** + * Converts value type string value into {@link SourceValueType} enum. + * + * @param stringValue value type string value + * @return source value type in optional container + */ + public static Optional fromValue(String stringValue) { + return Stream.of(values()) + .filter(keyType -> keyType.valueType.equalsIgnoreCase(stringValue)) + .findAny(); + } + + public static String getSupportedValueTypes() { + return Arrays.stream(SourceValueType.values()).map(SourceValueType::getValueType) + .collect(Collectors.joining(", ")); + } + + public String getValueType() { + return valueType; + } + + public String getValue() { + return value; + } +} diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/Test.java b/src/main/java/io/cdap/plugin/sfmc/source/util/Test.java new file mode 100644 index 0000000..a17e173 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/Test.java @@ -0,0 +1,99 @@ + +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package io.cdap.plugin.sfmc.source.util; + +import com.exacttarget.fuelsdk.ETClient; +import com.exacttarget.fuelsdk.ETConfiguration; +import com.exacttarget.fuelsdk.ETDataExtension; +import com.exacttarget.fuelsdk.ETDataExtensionRow; +import com.exacttarget.fuelsdk.ETFilter; +import com.exacttarget.fuelsdk.ETResponse; +import com.exacttarget.fuelsdk.ETSdkException; +import io.cdap.plugin.sfmc.source.SalesforceSourceConfig; + +import java.util.List; + +/** + * TokenGenerator class. + */ +public class Test { + public static void main(String[] args) + throws ETSdkException { + SalesforceSourceConfig conf = null; + TokenGenerator tokenGenerator = new TokenGenerator(); + long m = System.currentTimeMillis() / 1000; + + tokenGenerator.getAPIDetails(conf); + long n = System.currentTimeMillis() / 1000; + + System.out.println("Total second required" + (n - m)); + //ETClient client = new ETClient("/Users/akhilesh.trivedi/gitcdap/FuelSDK-Java-Example/src/main/java/com + // /exacttarget/fuelsdk/test/fuelsdk.properties"); + /* + ETResponse response = client.retrieve(ETFolder.class); + for (ETFolder folder : response.getObjects()) { + System.out.println(folder); + } + */ + // ETConfiguration configuration = new ETConfiguration(); + /* configuration.setClientId("isftmkzk5sr7olx6em78otnw"); + configuration.setClientSecret("OHT528TeEsDd5dHtrQOUjWtl"); + configuration.setSoapEndpoint("https://mc-67-bn30-84yc47k-pw5rw0vp1.soap.marketingcloudapis.com/Service.asmx"); + configuration.setEndpoint("https://mc-67-bn30-84yc47k-pw5rw0vp1.rest.marketingcloudapis.com/"); + configuration.setAuthEndpoint("https://mc-67-bn30-84yc47k-pw5rw0vp1.auth.marketingcloudapis.com/");*/ + /*configuration.set("clientId" , "isftmkzk5sr7olx6em78otnw"); + configuration.set("clientSecret" , "OHT528TeEsDd5dHtrQOUjWtl"); + configuration.set("soapEndpoint" , + "https://mc-67-bn30-84yc47k-pw5rw0vp1.soap.marketingcloudapis.com/Service.asmx"); + configuration.set("endpoint" , "https://mc-67-bn30-84yc47k-pw5rw0vp1.rest.marketingcloudapis.com/"); + configuration.set("authEndpoint" , "https://mc-67-bn30-84yc47k-pw5rw0vp1.auth.marketingcloudapis.com/"); + configuration.set("useOAuth2Authentication" , "true"); + ETClient client = new ETClient(configuration); + + ETFilter etf = new ETFilter(); + etf.addProperty("id"); + etf.addProperty("key"); + etf.addProperty("name"); + etf.addProperty("description"); + etf.addProperty("folderId"); + etf.addProperty("isSendable"); + etf.addProperty("isTestable"); +*/ + + /* Get list of all Data Extensions */ + + /* ETResponse dtresponse = client.retrieve(ETDataExtension.class,etf); + for (ETDataExtension dt : dtresponse.getObjects()) { + System.out.println("I am here--->"+dt); + ETResponse resp = dt.select(); + List rows = resp.getObjects(); + if ( rows.size() > 1 ) { + ETDataExtensionRow row1 = rows.get(0); + System.out.println("row--->"+row1); + } + }*/ + + /* Get Rows in each Data Extension + + ETResponse campaignETResponse = client.retrieve(ETCampaign.class,etf); + System.out.println(campaignETResponse.getResponseMessage()); + + */ + + + } +} diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/TokenGenerator.java b/src/main/java/io/cdap/plugin/sfmc/source/util/TokenGenerator.java new file mode 100644 index 0000000..5a1f7b2 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/TokenGenerator.java @@ -0,0 +1,90 @@ + +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.sfmc.source.util; + +import com.exacttarget.fuelsdk.ETClient; +import com.exacttarget.fuelsdk.ETConfiguration; +import com.exacttarget.fuelsdk.ETDataExtension; +import com.exacttarget.fuelsdk.ETDataExtensionRow; +import com.exacttarget.fuelsdk.ETFilter; +import com.exacttarget.fuelsdk.ETResponse; +import com.exacttarget.fuelsdk.ETSdkException; +import io.cdap.plugin.sfmc.source.SalesforceSourceConfig; +import io.cdap.plugin.sfmc.source.apiclient.SalesforceTableAPIClientImpl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + +/** + * TokenGenerator class. + */ +public class TokenGenerator { + private static final Logger LOG = LoggerFactory.getLogger(TokenGenerator.class); + + public ETClient getAPIDetails(SalesforceSourceConfig conf) throws ETSdkException { + + ETConfiguration configuration = new ETConfiguration(); + /* configuration.setClientId("isftmkzk5sr7olx6em78otnw"); + configuration.setClientSecret("OHT528TeEsDd5dHtrQOUjWtl"); + configuration.setSoapEndpoint("https://mc-67-bn30-84yc47k-pw5rw0vp1.soap.marketingcloudapis.com/Service.asmx"); + configuration.setEndpoint("https://mc-67-bn30-84yc47k-pw5rw0vp1.rest.marketingcloudapis.com/"); + configuration.setAuthEndpoint("https://mc-67-bn30-84yc47k-pw5rw0vp1.auth.marketingcloudapis.com/");*/ + /* configuration.set("clientId", conf.getClientId()); + configuration.set("clientSecret", conf.getClientSecret()); + configuration.set("soapEndpoint", conf.getSoapEndpoint()); + configuration.set("endpoint", conf.getRestEndpoint()); + configuration.set("authEndpoint", conf.getAuthEndpoint()); + configuration.set("useOAuth2Authentication", "true"); + ETClient client = new ETClient(configuration);*/ + configuration.set("clientId", "isftmkzk5sr7olx6em78otnw"); + configuration.set("clientSecret", "OHT528TeEsDd5dHtrQOUjWtl"); + configuration.set("soapEndpoint", + "https://mc-67-bn30-84yc47k-pw5rw0vp1.soap.marketingcloudapis.com/Service.asmx"); + configuration.set("endpoint", "https://mc-67-bn30-84yc47k-pw5rw0vp1.rest.marketingcloudapis.com/"); + configuration.set("authEndpoint", "https://mc-67-bn30-84yc47k-pw5rw0vp1.auth.marketingcloudapis.com/"); + configuration.set("useOAuth2Authentication", "true"); + ETClient client = new ETClient(configuration); + + /* ETFilter etf = new ETFilter(); + etf.addProperty("id"); + etf.addProperty("key"); + etf.addProperty("name"); + etf.addProperty("description"); + etf.addProperty("folderId"); + etf.addProperty("isSendable"); + etf.addProperty("isTestable");* + + + /* Get list of all Data Extensions */ + + /* ETResponse dtresponse = client.retrieve(ETDataExtension.class, etf); + for (ETDataExtension dt : dtresponse.getObjects()) { + System.out.println("ETDataExtension--->" + dt); + LOG.info("ETDataExtension--->" + dt); + ETResponse resp = dt.select(); + List rows = resp.getObjects(); + if (rows.size() > 1) { + ETDataExtensionRow row1 = rows.get(0); + LOG.info("row records--->" + row1); + } + }*/ + return client; + + } +} diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/Util.java b/src/main/java/io/cdap/plugin/sfmc/source/util/Util.java new file mode 100644 index 0000000..0bda13a --- /dev/null +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/Util.java @@ -0,0 +1,46 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.sfmc.source.util; + +import com.google.common.base.Strings; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; + +/** + * Utility class. + */ +public class Util { + public static boolean isNullOrEmpty(String string) { + return Strings.isNullOrEmpty(Strings.nullToEmpty(string).trim()); + } + + public static boolean isValidDateFormat(String dateFormat, String value) { + final SimpleDateFormat sdf = new SimpleDateFormat(dateFormat); + try { + final Date date = sdf.parse(value); + + if (!value.equals(sdf.format(date))) { + return false; + } + } catch (ParseException e) { + return false; + } + return true; + } +} diff --git a/widgets/Salesforce-batchsource.json b/widgets/Salesforce-batchsource.json new file mode 100644 index 0000000..6ef7473 --- /dev/null +++ b/widgets/Salesforce-batchsource.json @@ -0,0 +1,151 @@ +{ + "metadata": { + "spec-version": "1.0" + }, + "display-name": "Salesforce", + "configuration-groups": [ + { + "label": "Basic", + "properties": [ + { + "widget-type": "textbox", + "label": "Reference Name", + "name": "referenceName", + "widget-attributes": { + "placeholder": "Name used to identify this source for lineage" + } + }, + { + "widget-type": "select", + "label": "Data retrieval mode", + "name": "queryMode", + "widget-attributes": { + "values": [ + "Multi-Object", + "Single-Object" + ], + "default": "Single-object" + } + } + ] + }, + { + "label": "Single-Object Mode", + "properties": [ + { + "widget-type": "select", + "label": "Object Name", + "name": "objectName", + "widget-attributes": { + "values": [ + "DataExtension", + "Campaign", + "Account", + "Email" + ], + "default": "DataExtension" + } + } + ] + }, + { + "label": "Multi-Object Mode", + "properties": [ + { + "widget-type": "textbox", + "label": "List Of Object Name", + "name": "multiObjectName", + "widget-attributes": { + "placeholder": "Salesforce object name from which data to be fetched" + } + } + ] + }, + { + "label": "Credentials", + "properties": [ + { + "widget-type": "textbox", + "label": "Client ID", + "name": "clientId", + "widget-attributes" : { + "placeholder": "Salesforce Client ID" + } + }, + { + "widget-type": "password", + "label": "Client Secret", + "name": "clientSecret", + "widget-attributes" : { + "placeholder": "Salesforce Client Secret Key" + } + }, + { + "widget-type": "textbox", + "label": "AUTH API Endpoint", + "name": "authEndpoint", + "widget-attributes" : { + "placeholder": "Salesforce REST API Endpoint e.g. https://instance.auth.marketingcloudapis.com" + } + }, + { + "widget-type": "textbox", + "label": "SOAP API Endpoint", + "name": "soapEndpoint", + "widget-attributes" : { + "placeholder": "Salesforce SOAP API Endpoint e.g. https://instance.Salesforce.soap.marketingcloudapis.com/Service.asmx" + } + }, + { + "widget-type": "textbox", + "label": "REST API Endpoint", + "name": "restEndpoint", + "widget-attributes" : { + "placeholder": "Salesforce REST API Endpoint e.g. https://instance.rest.marketingcloudapis.com" + } + } + ] + }, + { + "label": "Advanced", + "properties": [ + { + "widget-type": "select", + "label": "Type of values", + "name": "valueType", + "widget-attributes": { + "values": [ + "Actual", + "Display" + ], + "default": "Actual" + } + }, + { + "widget-type": "textbox", + "label": "Start Date", + "name": "startDate", + "widget-attributes" : { + "placeholder": "Start date to be used to filter the data" + } + }, + { + "widget-type": "textbox", + "label": "End Date", + "name": "endDate", + "widget-attributes" : { + "placeholder": "End date to be used to filter the data" + } + } + ] + } + ], + "outputs": [ ], + "jump-config": { + "datasets": [ + { + "ref-property-name": "referenceName" + } + ] + } +} From 79f806ecdc30bf5ee7a0ef892009d69f940b8ef9 Mon Sep 17 00:00:00 2001 From: "akhilesh.trivedi" Date: Wed, 18 Mar 2020 15:43:22 +0530 Subject: [PATCH 02/19] Code has been reverted for DataExtensionClient --- .../cdap/plugin/sfmc/DataExtensionClient.java | 42 +++++++------------ 1 file changed, 14 insertions(+), 28 deletions(-) diff --git a/src/main/java/io/cdap/plugin/sfmc/DataExtensionClient.java b/src/main/java/io/cdap/plugin/sfmc/DataExtensionClient.java index 7d605f5..e4b875d 100644 --- a/src/main/java/io/cdap/plugin/sfmc/DataExtensionClient.java +++ b/src/main/java/io/cdap/plugin/sfmc/DataExtensionClient.java @@ -107,8 +107,7 @@ public void validateSchemaCompatibility(Schema schema, FailureCollector collecto if (schemaField == null) { if (column.getIsRequired()) { collector.addFailure( - String.format("Data extension '%s' contains a required column '%s' of type '%s'" + - " that is not present in " + + String.format("Data extension '%s' contains a required column '%s' of type '%s' that is not present in " + "the input schema.", dataExtensionKey, columnName, column.getType()) , null) .withConfigProperty(MarketingCloudConf.DATA_EXTENSION); @@ -131,22 +130,18 @@ public void validateSchemaCompatibility(Schema schema, FailureCollector collecto case BOOLEAN: if (schemaType != Schema.Type.BOOLEAN) { collector.addFailure( - String.format("Column '%s' is a boolean in data extension '%s', but is a '%s'" + - " in the input schema.", + String.format("Column '%s' is a boolean in data extension '%s', but is a '%s' in the input schema.", originalName, dataExtensionKey, schemaTypeStr), - "Change the field schema to ensure it is" + - " a boolean or string type.") + "Change the field schema to ensure it is a boolean or string type.") .withInputSchemaField(fieldName); } break; case DECIMAL: if (fieldSchema.getLogicalType() != Schema.LogicalType.DECIMAL) { collector.addFailure( - String.format("Column '%s' is a decimal in data extension '%s'," + - " but is a '%s' in the input schema.", + String.format("Column '%s' is a decimal in data extension '%s', but is a '%s' in the input schema.", originalName, dataExtensionKey, schemaTypeStr), - "Change the field schema to ensure it is a decimal" + - " or string type.").withInputSchemaField(fieldName); + "Change the field schema to ensure it is a decimal or string type.").withInputSchemaField(fieldName); } break; case PHONE: @@ -154,8 +149,7 @@ public void validateSchemaCompatibility(Schema schema, FailureCollector collecto case EMAIL_ADDRESS: case LOCALE: collector.addFailure( - String.format("Column '%s' is a %s in data extension '%s', but is a '%s'" + - " in the input schema.", + String.format("Column '%s' is a %s in data extension '%s', but is a '%s' in the input schema.", originalName, column.getType().name().toLowerCase(), dataExtensionKey, schemaTypeStr), "Change the field schema to ensure it is a string type.") @@ -164,8 +158,7 @@ public void validateSchemaCompatibility(Schema schema, FailureCollector collecto case DATE: if (fieldSchema.getLogicalType() != Schema.LogicalType.DATE) { collector.addFailure( - String.format("Column '%s' is a date in data extension '%s', but is a '%s'" + - " in the input schema.", + String.format("Column '%s' is a date in data extension '%s', but is a '%s' in the input schema.", originalName, dataExtensionKey, schemaTypeStr), "Change the field schema to ensure it is a date or string type." ).withInputSchemaField(fieldName); @@ -174,19 +167,16 @@ public void validateSchemaCompatibility(Schema schema, FailureCollector collecto case NUMBER: if (schemaType != Schema.Type.INT) { collector.addFailure( - String.format("Column '%s' is a number in data extension '%s', but is a '%s'" + - " in the input schema.", + String.format("Column '%s' is a number in data extension '%s', but is a '%s' in the input schema.", originalName, dataExtensionKey, schemaTypeStr), - "Change the field schema to ensure it is an integer or string" + - " type.").withInputSchemaField(fieldName); + "Change the field schema to ensure it is an integer or string type.").withInputSchemaField(fieldName); } break; default: collector.addFailure( String.format("Unknown type '%s' for column '%s' in data extension '%s'.", column.getType(), column.getName(), dataExtensionKey), - "Supported types are: boolean, decimal, phone, text, email_address," + - " locale, date and number.") + "Supported types are: boolean, decimal, phone, text, email_address, locale, date and number.") .withInputSchemaField(fieldName); } } @@ -490,21 +480,17 @@ public static ETResponse update(ETClient client, * Used to run the client from the command line. Primarily used for testing purposes. * Expects arguments in the form [command] [key] [properties files] where properties file is expected * to contain lines like: - *

+ * * clientId=... * clientSecret=... * authEndpoint=... * soapEndpoint=... */ public static void main(String[] args) throws Exception { - /* String command = args[0]; + String command = args[0]; String key = args[1]; - String propertiesFile = args[2];*/ - String command = "describe"; - // String key ="2E2DF3B8-F3A0-415C-9EFC-922D32E13E61"; - String key = "7CBE457F-F056-4154-B21F-647CD11B81CF"; - ETConfiguration conf = new ETConfiguration("/Users/akhilesh.trivedi/gitcdap/FuelSDK-Java-Example" + - "/src/main/java/com/exacttarget/fuelsdk/test/fuelsdk.properties"); + String propertiesFile = args[2]; + ETConfiguration conf = new ETConfiguration(propertiesFile); DataExtensionClient client = new DataExtensionClient(new ETClient(conf), key); From 0b5925cc578db005b9ce61f1b91279a533c2a3d5 Mon Sep 17 00:00:00 2001 From: "akhilesh.trivedi" Date: Wed, 18 Mar 2020 18:46:40 +0530 Subject: [PATCH 03/19] Connection is being created --- .../sfmc/source/SalesforceInputFormat.java | 4 +- .../sfmc/source/SalesforceInputSplit.java | 2 +- .../sfmc/source/SalesforceRecordReader.java | 2 +- .../plugin/sfmc/source/SalesforceSource.java | 9 +- .../SalesforceTableAPIClientImpl.java | 22 ++--- .../SalesforceTableDataResponse.java | 2 +- .../sfmc/source/util/SalesforceColumn.java | 2 +- .../sfmc/source/util/SalesforceConstants.java | 4 +- .../source/util/SalesforceObjectInfo.java | 2 +- .../sfmc/source/util/SourceApplication.java | 2 +- .../sfmc/source/util/SourceObjectMode.java | 2 +- .../sfmc/source/util/SourceValueType.java | 6 +- ...kenGenerator.java => TableConnection.java} | 37 ++++--- .../io/cdap/plugin/sfmc/source/util/Test.java | 99 ------------------- 14 files changed, 52 insertions(+), 143 deletions(-) rename src/main/java/io/cdap/plugin/sfmc/source/util/{TokenGenerator.java => TableConnection.java} (79%) delete mode 100644 src/main/java/io/cdap/plugin/sfmc/source/util/Test.java diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java index f1f3eea..982567f 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java @@ -43,13 +43,13 @@ import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PAGE_SIZE; /** - * ServiceNow input format + * Salesforce input format */ public class SalesforceInputFormat extends InputFormat { private static final Logger LOG = LoggerFactory.getLogger(SalesforceInputFormat.class); /** - * Configure the input format to read tables from ServiceNow. Should be called from the mapreduce client. + * Configure the input format to read tables from Salesforce. Should be called from the mapreduce client. * * @param jobConfig the job configuration * @param mode diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputSplit.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputSplit.java index 2e0c8d7..b1e2749 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputSplit.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputSplit.java @@ -24,7 +24,7 @@ import java.io.IOException; /** - * ServiceNow input split + * Salesforce input split */ public class SalesforceInputSplit extends InputSplit implements Writable { private String tableName; diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java index dcbd5f3..6be0b32 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java @@ -43,7 +43,7 @@ import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PAGE_SIZE; /** - * Record reader that reads the entire contents of a ServiceNow table. + * Record reader that reads the entire contents of a Salesforce table. */ public class SalesforceRecordReader extends RecordReader { private static final Logger LOG = LoggerFactory.getLogger(SalesforceRecordReader.class); diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java index bab926e..fd2524a 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java @@ -19,19 +19,18 @@ import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.annotation.Plugin; -import io.cdap.cdap.api.data.batch.Input; + import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.dataset.lib.KeyValue; import io.cdap.cdap.etl.api.Emitter; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.PipelineConfigurer; import io.cdap.cdap.etl.api.StageConfigurer; -import io.cdap.cdap.etl.api.action.SettableArguments; + import io.cdap.cdap.etl.api.batch.BatchSource; import io.cdap.cdap.etl.api.batch.BatchSourceContext; -import io.cdap.plugin.common.SourceInputFormatProvider; -//import io.cdap.plugin.servicenow.source.util.SalesforceObjectInfo; -//import io.cdap.plugin.servicenow.source.util.SourceObjectMode; + + import io.cdap.plugin.sfmc.source.util.SalesforceObjectInfo; import io.cdap.plugin.sfmc.source.util.SourceObjectMode; import org.apache.hadoop.conf.Configuration; diff --git a/src/main/java/io/cdap/plugin/sfmc/source/apiclient/SalesforceTableAPIClientImpl.java b/src/main/java/io/cdap/plugin/sfmc/source/apiclient/SalesforceTableAPIClientImpl.java index d80ef31..b4d484a 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/apiclient/SalesforceTableAPIClientImpl.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/apiclient/SalesforceTableAPIClientImpl.java @@ -16,28 +16,27 @@ package io.cdap.plugin.sfmc.source.apiclient; -import com.exacttarget.fuelsdk.ETClient; -import com.exacttarget.fuelsdk.ETSdkException; -import com.google.common.base.Strings; + + import com.google.gson.Gson; import com.google.gson.JsonArray; import com.google.gson.JsonObject; import com.google.gson.reflect.TypeToken; +import io.cdap.plugin.sfmc.DataExtensionClient; import io.cdap.plugin.sfmc.source.SalesforceSourceConfig; import io.cdap.plugin.sfmc.source.util.SalesforceColumn; -import io.cdap.plugin.sfmc.source.util.TokenGenerator; +import io.cdap.plugin.sfmc.source.util.TableConnection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.lang.reflect.Type; -import java.util.ArrayList; -import java.util.Collections; + import java.util.List; import java.util.Map; /** - * Implementation class for ServiceNow Table API + * Implementation class for Salesforce Table API */ public class SalesforceTableAPIClientImpl { private static final Logger LOG = LoggerFactory.getLogger(SalesforceTableAPIClientImpl.class); @@ -48,11 +47,12 @@ public class SalesforceTableAPIClientImpl { public SalesforceTableAPIClientImpl(SalesforceSourceConfig conf) { this.conf = conf; - TokenGenerator tokenGenerator = new TokenGenerator(); + TableConnection tableConnection = new TableConnection(); try { - ETClient client = tokenGenerator.getAPIDetails(conf); - client.getAccessToken(); - LOG.info("token{}, client{}", client.getAccessToken(), client.getClientId()); + DataExtensionClient client = tableConnection.getTableConnection(conf); + + LOG.info("getDataExtensionInfo {}, getDataExtensionKey {}", + client.getDataExtensionInfo(), client.getDataExtensionKey()); } catch (Exception ets) { ets.printStackTrace(); } diff --git a/src/main/java/io/cdap/plugin/sfmc/source/apiclient/SalesforceTableDataResponse.java b/src/main/java/io/cdap/plugin/sfmc/source/apiclient/SalesforceTableDataResponse.java index afa159a..6426ecd 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/apiclient/SalesforceTableDataResponse.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/apiclient/SalesforceTableDataResponse.java @@ -24,7 +24,7 @@ import java.util.Map; /** - * A Pojo class to wrap the success response for ServiceNow Table data + * A Pojo class to wrap the success response for Salesforce Table data */ public class SalesforceTableDataResponse { private int totalRecordCount; diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceColumn.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceColumn.java index 23a31a1..d9f6913 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceColumn.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceColumn.java @@ -20,7 +20,7 @@ import com.google.gson.annotations.SerializedName; /** - * Information about a ServiceNow table column. + * Information about a Salesforce table column. */ public class SalesforceColumn { @SerializedName("element") diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java index a9be2cf..82cc3d2 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java @@ -17,12 +17,12 @@ package io.cdap.plugin.sfmc.source.util; /** - * ServiceNow constants. + * Salesforce constants. */ public interface SalesforceConstants { /** - * ServiceNow plugin name. + * Salesforce plugin name. */ String PLUGIN_NAME = "Salesforce"; diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceObjectInfo.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceObjectInfo.java index 64b8ab8..1eba8ed 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceObjectInfo.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceObjectInfo.java @@ -19,7 +19,7 @@ import io.cdap.cdap.api.data.schema.Schema; /** - * Information about a ServiceNow table. + * Information about a Salesforce table. */ public class SalesforceObjectInfo { private final String tableName; diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SourceApplication.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SourceApplication.java index d8dfa24..7f9061e 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SourceApplication.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SourceApplication.java @@ -23,7 +23,7 @@ import java.util.stream.Stream; /** - * Indicates application for which data to be fetched from multiple tables in ServiceNow. + * Indicates application for which data to be fetched from multiple tables in Salesforce. */ public enum SourceApplication { diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SourceObjectMode.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SourceObjectMode.java index 92ac468..fd179d5 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SourceObjectMode.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SourceObjectMode.java @@ -22,7 +22,7 @@ import java.util.stream.Stream; /** - * Indicates query mode which will be used when fetching ServiceNow data. + * Indicates query mode which will be used when fetching Salesforce data. */ public enum SourceObjectMode { diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SourceValueType.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SourceValueType.java index 65fb1e2..f1aec40 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SourceValueType.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SourceValueType.java @@ -22,17 +22,17 @@ import java.util.stream.Stream; /** - * Indicates type of value to be returned from ServiceNow Table API + * Indicates type of value to be returned from Salesforce Table API */ public enum SourceValueType { /** - * Actual values will be returned by ServiceNow Table API + * Actual values will be returned by Salesforce Table API */ SHOW_ACTUAL_VALUE("Actual", "false"), /** - * Display values will be returned by ServiceNow Table API + * Display values will be returned by Salesforce Table API */ SHOW_DISPLAY_VALUE("Display", "true"); diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/TokenGenerator.java b/src/main/java/io/cdap/plugin/sfmc/source/util/TableConnection.java similarity index 79% rename from src/main/java/io/cdap/plugin/sfmc/source/util/TokenGenerator.java rename to src/main/java/io/cdap/plugin/sfmc/source/util/TableConnection.java index 5a1f7b2..447932c 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/TokenGenerator.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/TableConnection.java @@ -17,28 +17,24 @@ package io.cdap.plugin.sfmc.source.util; -import com.exacttarget.fuelsdk.ETClient; import com.exacttarget.fuelsdk.ETConfiguration; -import com.exacttarget.fuelsdk.ETDataExtension; -import com.exacttarget.fuelsdk.ETDataExtensionRow; -import com.exacttarget.fuelsdk.ETFilter; -import com.exacttarget.fuelsdk.ETResponse; import com.exacttarget.fuelsdk.ETSdkException; +import io.cdap.plugin.sfmc.DataExtensionClient; import io.cdap.plugin.sfmc.source.SalesforceSourceConfig; -import io.cdap.plugin.sfmc.source.apiclient.SalesforceTableAPIClientImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.List; +import java.io.IOException; /** - * TokenGenerator class. + * TableConnection class. + * A data extension is a table that contains your data */ -public class TokenGenerator { - private static final Logger LOG = LoggerFactory.getLogger(TokenGenerator.class); - - public ETClient getAPIDetails(SalesforceSourceConfig conf) throws ETSdkException { +public class TableConnection { + private static final Logger LOG = LoggerFactory.getLogger(TableConnection.class); + public DataExtensionClient getTableConnection(SalesforceSourceConfig conf) { + DataExtensionClient client = null; ETConfiguration configuration = new ETConfiguration(); /* configuration.setClientId("isftmkzk5sr7olx6em78otnw"); configuration.setClientSecret("OHT528TeEsDd5dHtrQOUjWtl"); @@ -51,7 +47,7 @@ public ETClient getAPIDetails(SalesforceSourceConfig conf) throws ETSdkException configuration.set("endpoint", conf.getRestEndpoint()); configuration.set("authEndpoint", conf.getAuthEndpoint()); configuration.set("useOAuth2Authentication", "true"); - ETClient client = new ETClient(configuration);*/ + ETClient client = new ETClient(configuration); configuration.set("clientId", "isftmkzk5sr7olx6em78otnw"); configuration.set("clientSecret", "OHT528TeEsDd5dHtrQOUjWtl"); configuration.set("soapEndpoint", @@ -59,7 +55,20 @@ public ETClient getAPIDetails(SalesforceSourceConfig conf) throws ETSdkException configuration.set("endpoint", "https://mc-67-bn30-84yc47k-pw5rw0vp1.rest.marketingcloudapis.com/"); configuration.set("authEndpoint", "https://mc-67-bn30-84yc47k-pw5rw0vp1.auth.marketingcloudapis.com/"); configuration.set("useOAuth2Authentication", "true"); - ETClient client = new ETClient(configuration); + ETClient client = new ETClient(configuration);*/ + try { + client = DataExtensionClient.create("2E2DF3B8-F3A0-415C-9EFC-922D32E13E61", + conf.getClientId(), conf.getClientSecret(), + conf.getAuthEndpoint(), conf.getSoapEndpoint()); + + } catch (ETSdkException e) { + try { + throw new IOException("Unable to create Salesforce Marketing Cloud client.", e); + } catch (IOException e1) { + e1.printStackTrace(); + } + } + /* ETFilter etf = new ETFilter(); etf.addProperty("id"); diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/Test.java b/src/main/java/io/cdap/plugin/sfmc/source/util/Test.java deleted file mode 100644 index a17e173..0000000 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/Test.java +++ /dev/null @@ -1,99 +0,0 @@ - -/* - * Copyright © 2019 Cask Data, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package io.cdap.plugin.sfmc.source.util; - -import com.exacttarget.fuelsdk.ETClient; -import com.exacttarget.fuelsdk.ETConfiguration; -import com.exacttarget.fuelsdk.ETDataExtension; -import com.exacttarget.fuelsdk.ETDataExtensionRow; -import com.exacttarget.fuelsdk.ETFilter; -import com.exacttarget.fuelsdk.ETResponse; -import com.exacttarget.fuelsdk.ETSdkException; -import io.cdap.plugin.sfmc.source.SalesforceSourceConfig; - -import java.util.List; - -/** - * TokenGenerator class. - */ -public class Test { - public static void main(String[] args) - throws ETSdkException { - SalesforceSourceConfig conf = null; - TokenGenerator tokenGenerator = new TokenGenerator(); - long m = System.currentTimeMillis() / 1000; - - tokenGenerator.getAPIDetails(conf); - long n = System.currentTimeMillis() / 1000; - - System.out.println("Total second required" + (n - m)); - //ETClient client = new ETClient("/Users/akhilesh.trivedi/gitcdap/FuelSDK-Java-Example/src/main/java/com - // /exacttarget/fuelsdk/test/fuelsdk.properties"); - /* - ETResponse response = client.retrieve(ETFolder.class); - for (ETFolder folder : response.getObjects()) { - System.out.println(folder); - } - */ - // ETConfiguration configuration = new ETConfiguration(); - /* configuration.setClientId("isftmkzk5sr7olx6em78otnw"); - configuration.setClientSecret("OHT528TeEsDd5dHtrQOUjWtl"); - configuration.setSoapEndpoint("https://mc-67-bn30-84yc47k-pw5rw0vp1.soap.marketingcloudapis.com/Service.asmx"); - configuration.setEndpoint("https://mc-67-bn30-84yc47k-pw5rw0vp1.rest.marketingcloudapis.com/"); - configuration.setAuthEndpoint("https://mc-67-bn30-84yc47k-pw5rw0vp1.auth.marketingcloudapis.com/");*/ - /*configuration.set("clientId" , "isftmkzk5sr7olx6em78otnw"); - configuration.set("clientSecret" , "OHT528TeEsDd5dHtrQOUjWtl"); - configuration.set("soapEndpoint" , - "https://mc-67-bn30-84yc47k-pw5rw0vp1.soap.marketingcloudapis.com/Service.asmx"); - configuration.set("endpoint" , "https://mc-67-bn30-84yc47k-pw5rw0vp1.rest.marketingcloudapis.com/"); - configuration.set("authEndpoint" , "https://mc-67-bn30-84yc47k-pw5rw0vp1.auth.marketingcloudapis.com/"); - configuration.set("useOAuth2Authentication" , "true"); - ETClient client = new ETClient(configuration); - - ETFilter etf = new ETFilter(); - etf.addProperty("id"); - etf.addProperty("key"); - etf.addProperty("name"); - etf.addProperty("description"); - etf.addProperty("folderId"); - etf.addProperty("isSendable"); - etf.addProperty("isTestable"); -*/ - - /* Get list of all Data Extensions */ - - /* ETResponse dtresponse = client.retrieve(ETDataExtension.class,etf); - for (ETDataExtension dt : dtresponse.getObjects()) { - System.out.println("I am here--->"+dt); - ETResponse resp = dt.select(); - List rows = resp.getObjects(); - if ( rows.size() > 1 ) { - ETDataExtensionRow row1 = rows.get(0); - System.out.println("row--->"+row1); - } - }*/ - - /* Get Rows in each Data Extension - - ETResponse campaignETResponse = client.retrieve(ETCampaign.class,etf); - System.out.println(campaignETResponse.getResponseMessage()); - - */ - - - } -} From 90146db08c1caa59d95c166658f4713a051ff381 Mon Sep 17 00:00:00 2001 From: "akhilesh.trivedi" Date: Thu, 19 Mar 2020 16:12:33 +0530 Subject: [PATCH 04/19] Removed unwanted code --- .../sfmc/source/util/TableConnection.java | 51 ++----------------- 1 file changed, 3 insertions(+), 48 deletions(-) diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/TableConnection.java b/src/main/java/io/cdap/plugin/sfmc/source/util/TableConnection.java index 447932c..3be965a 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/TableConnection.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/TableConnection.java @@ -33,32 +33,11 @@ public class TableConnection { private static final Logger LOG = LoggerFactory.getLogger(TableConnection.class); - public DataExtensionClient getTableConnection(SalesforceSourceConfig conf) { + public DataExtensionClient getTableConnection(SalesforceSourceConfig conf) { DataExtensionClient client = null; - ETConfiguration configuration = new ETConfiguration(); - /* configuration.setClientId("isftmkzk5sr7olx6em78otnw"); - configuration.setClientSecret("OHT528TeEsDd5dHtrQOUjWtl"); - configuration.setSoapEndpoint("https://mc-67-bn30-84yc47k-pw5rw0vp1.soap.marketingcloudapis.com/Service.asmx"); - configuration.setEndpoint("https://mc-67-bn30-84yc47k-pw5rw0vp1.rest.marketingcloudapis.com/"); - configuration.setAuthEndpoint("https://mc-67-bn30-84yc47k-pw5rw0vp1.auth.marketingcloudapis.com/");*/ - /* configuration.set("clientId", conf.getClientId()); - configuration.set("clientSecret", conf.getClientSecret()); - configuration.set("soapEndpoint", conf.getSoapEndpoint()); - configuration.set("endpoint", conf.getRestEndpoint()); - configuration.set("authEndpoint", conf.getAuthEndpoint()); - configuration.set("useOAuth2Authentication", "true"); - ETClient client = new ETClient(configuration); - configuration.set("clientId", "isftmkzk5sr7olx6em78otnw"); - configuration.set("clientSecret", "OHT528TeEsDd5dHtrQOUjWtl"); - configuration.set("soapEndpoint", - "https://mc-67-bn30-84yc47k-pw5rw0vp1.soap.marketingcloudapis.com/Service.asmx"); - configuration.set("endpoint", "https://mc-67-bn30-84yc47k-pw5rw0vp1.rest.marketingcloudapis.com/"); - configuration.set("authEndpoint", "https://mc-67-bn30-84yc47k-pw5rw0vp1.auth.marketingcloudapis.com/"); - configuration.set("useOAuth2Authentication", "true"); - ETClient client = new ETClient(configuration);*/ try { - client = DataExtensionClient.create("2E2DF3B8-F3A0-415C-9EFC-922D32E13E61", - conf.getClientId(), conf.getClientSecret(), + client = DataExtensionClient.create("2E2DF3B8-F3A0-415C-9EFC-922D32E13E61", + conf.getClientId(), conf.getClientSecret(), conf.getAuthEndpoint(), conf.getSoapEndpoint()); } catch (ETSdkException e) { @@ -69,30 +48,6 @@ public DataExtensionClient getTableConnection(SalesforceSourceConfig conf) { } } - - /* ETFilter etf = new ETFilter(); - etf.addProperty("id"); - etf.addProperty("key"); - etf.addProperty("name"); - etf.addProperty("description"); - etf.addProperty("folderId"); - etf.addProperty("isSendable"); - etf.addProperty("isTestable");* - - - /* Get list of all Data Extensions */ - - /* ETResponse dtresponse = client.retrieve(ETDataExtension.class, etf); - for (ETDataExtension dt : dtresponse.getObjects()) { - System.out.println("ETDataExtension--->" + dt); - LOG.info("ETDataExtension--->" + dt); - ETResponse resp = dt.select(); - List rows = resp.getObjects(); - if (rows.size() > 1) { - ETDataExtensionRow row1 = rows.get(0); - LOG.info("row records--->" + row1); - } - }*/ return client; } From 6299d65a70f7c6bed1e1f6cdc3994a2670495bb1 Mon Sep 17 00:00:00 2001 From: "akhilesh.trivedi" Date: Thu, 19 Mar 2020 16:30:40 +0530 Subject: [PATCH 05/19] TableConnection class has been deleted --- .../SalesforceTableAPIClientImpl.java | 7 +-- .../sfmc/source/util/TableConnection.java | 54 ------------------- 2 files changed, 4 insertions(+), 57 deletions(-) delete mode 100644 src/main/java/io/cdap/plugin/sfmc/source/util/TableConnection.java diff --git a/src/main/java/io/cdap/plugin/sfmc/source/apiclient/SalesforceTableAPIClientImpl.java b/src/main/java/io/cdap/plugin/sfmc/source/apiclient/SalesforceTableAPIClientImpl.java index b4d484a..30db0fc 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/apiclient/SalesforceTableAPIClientImpl.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/apiclient/SalesforceTableAPIClientImpl.java @@ -26,7 +26,6 @@ import io.cdap.plugin.sfmc.DataExtensionClient; import io.cdap.plugin.sfmc.source.SalesforceSourceConfig; import io.cdap.plugin.sfmc.source.util.SalesforceColumn; -import io.cdap.plugin.sfmc.source.util.TableConnection; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,9 +46,11 @@ public class SalesforceTableAPIClientImpl { public SalesforceTableAPIClientImpl(SalesforceSourceConfig conf) { this.conf = conf; - TableConnection tableConnection = new TableConnection(); + try { - DataExtensionClient client = tableConnection.getTableConnection(conf); + DataExtensionClient client = client = DataExtensionClient.create("DATAEXTENSIONKEY", + conf.getClientId(), conf.getClientSecret(), + conf.getAuthEndpoint(), conf.getSoapEndpoint()); LOG.info("getDataExtensionInfo {}, getDataExtensionKey {}", client.getDataExtensionInfo(), client.getDataExtensionKey()); diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/TableConnection.java b/src/main/java/io/cdap/plugin/sfmc/source/util/TableConnection.java deleted file mode 100644 index 3be965a..0000000 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/TableConnection.java +++ /dev/null @@ -1,54 +0,0 @@ - -/* - * Copyright © 2019 Cask Data, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package io.cdap.plugin.sfmc.source.util; - -import com.exacttarget.fuelsdk.ETConfiguration; -import com.exacttarget.fuelsdk.ETSdkException; -import io.cdap.plugin.sfmc.DataExtensionClient; -import io.cdap.plugin.sfmc.source.SalesforceSourceConfig; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; - -/** - * TableConnection class. - * A data extension is a table that contains your data - */ -public class TableConnection { - private static final Logger LOG = LoggerFactory.getLogger(TableConnection.class); - - public DataExtensionClient getTableConnection(SalesforceSourceConfig conf) { - DataExtensionClient client = null; - try { - client = DataExtensionClient.create("2E2DF3B8-F3A0-415C-9EFC-922D32E13E61", - conf.getClientId(), conf.getClientSecret(), - conf.getAuthEndpoint(), conf.getSoapEndpoint()); - - } catch (ETSdkException e) { - try { - throw new IOException("Unable to create Salesforce Marketing Cloud client.", e); - } catch (IOException e1) { - e1.printStackTrace(); - } - } - - return client; - - } -} From 81b0d4d48582924f365eebaf795f69d176c5ea38 Mon Sep 17 00:00:00 2001 From: amolkokane76 Date: Thu, 19 Mar 2020 22:11:21 +0530 Subject: [PATCH 06/19] Implemented data extensions source plugin --- ...> SalesforceDataExtension-batchsource.png} | Bin pom.xml | 19 +- .../cdap/plugin/sfmc/DataExtensionClient.java | 515 ----------------- .../sfmc/DataExtensionOutputFormat.java | 1 + .../sfmc/DataExtensionRecordWriter.java | 1 + .../cdap/plugin/sfmc/MarketingCloudConf.java | 1 + .../sfmc/RecordDataExtensionRowConverter.java | 3 +- .../sfmc/common/DataExtensionClient.java | 541 ++++++++++++++++++ .../sfmc/{ => common}/DataExtensionInfo.java | 2 +- .../sfmc/source/SalesforceInputFormat.java | 75 ++- .../sfmc/source/SalesforceInputSplit.java | 8 +- .../sfmc/source/SalesforceRecordReader.java | 100 ++-- .../plugin/sfmc/source/SalesforceSource.java | 32 +- .../sfmc/source/SalesforceSourceConfig.java | 205 +++---- .../SalesforceTableAPIClientImpl.java | 2 +- .../sfmc/source/util/SalesforceConstants.java | 34 +- .../source/util/SalesforceObjectInfo.java | 8 +- .../sfmc/source/util/SchemaBuilder.java | 23 +- .../sfmc/source/util/SourceObjectMode.java | 4 +- .../sfmc/source/util/TableConnection.java | 2 +- .../io/cdap/plugin/sfmc/source/util/Util.java | 19 +- ... SalesforceDataExtension-batchsource.json} | 86 ++- 22 files changed, 837 insertions(+), 844 deletions(-) rename icons/{Salesforce-batchsource.png => SalesforceDataExtension-batchsource.png} (100%) delete mode 100644 src/main/java/io/cdap/plugin/sfmc/DataExtensionClient.java create mode 100644 src/main/java/io/cdap/plugin/sfmc/common/DataExtensionClient.java rename src/main/java/io/cdap/plugin/sfmc/{ => common}/DataExtensionInfo.java (97%) rename widgets/{Salesforce-batchsource.json => SalesforceDataExtension-batchsource.json} (59%) diff --git a/icons/Salesforce-batchsource.png b/icons/SalesforceDataExtension-batchsource.png similarity index 100% rename from icons/Salesforce-batchsource.png rename to icons/SalesforceDataExtension-batchsource.png diff --git a/pom.xml b/pom.xml index 7c716e0..0b9e527 100644 --- a/pom.xml +++ b/pom.xml @@ -30,10 +30,12 @@ true UTF-8 - 6.1.0-SNAPSHOT + 6.1.1 + 2.3.4 4.12 2.3.0 2.2.0 + 27.0.1-jre 1.4.0 @@ -58,21 +60,24 @@ io.cdap.cdap - cdap-api + cdap-etl-api ${cdap.version} - - io.cdap.cdap - cdap-etl-api - ${cdap.version} - + io.cdap.plugin + hydrator-common + ${cdap.plugin.version} io.cdap.plugin hydrator-common ${hydrator.version} + + com.google.guava + guava + ${guava.version} + org.apache.hadoop hadoop-common diff --git a/src/main/java/io/cdap/plugin/sfmc/DataExtensionClient.java b/src/main/java/io/cdap/plugin/sfmc/DataExtensionClient.java deleted file mode 100644 index e4b875d..0000000 --- a/src/main/java/io/cdap/plugin/sfmc/DataExtensionClient.java +++ /dev/null @@ -1,515 +0,0 @@ -/* - * Copyright © 2019 Cask Data, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package io.cdap.plugin.sfmc; - -import com.exacttarget.fuelsdk.ETClient; -import com.exacttarget.fuelsdk.ETConfiguration; -import com.exacttarget.fuelsdk.ETDataExtension; -import com.exacttarget.fuelsdk.ETDataExtensionColumn; -import com.exacttarget.fuelsdk.ETDataExtensionRow; -import com.exacttarget.fuelsdk.ETResponse; -import com.exacttarget.fuelsdk.ETResult; -import com.exacttarget.fuelsdk.ETSdkException; -import com.exacttarget.fuelsdk.ETSoapConnection; -import com.exacttarget.fuelsdk.ETSoapObject; -import com.exacttarget.fuelsdk.internal.APIObject; -import com.exacttarget.fuelsdk.internal.CreateOptions; -import com.exacttarget.fuelsdk.internal.CreateRequest; -import com.exacttarget.fuelsdk.internal.CreateResponse; -import com.exacttarget.fuelsdk.internal.CreateResult; -import com.exacttarget.fuelsdk.internal.DataExtensionCreateResult; -import com.exacttarget.fuelsdk.internal.DataExtensionUpdateResult; -import com.exacttarget.fuelsdk.internal.Soap; -import com.exacttarget.fuelsdk.internal.UpdateOptions; -import com.exacttarget.fuelsdk.internal.UpdateRequest; -import com.exacttarget.fuelsdk.internal.UpdateResponse; -import com.exacttarget.fuelsdk.internal.UpdateResult; -import io.cdap.cdap.api.data.schema.Schema; -import io.cdap.cdap.etl.api.FailureCollector; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * Wrapper around an ETClient that understands objects at the level that the plugin cares about. - */ -public class DataExtensionClient { - private final ETClient client; - private final String dataExtensionKey; - - DataExtensionClient(ETClient client, String dataExtensionKey) { - this.client = client; - this.dataExtensionKey = dataExtensionKey; - } - - public static DataExtensionClient create(String dataExtensionKey, String clientId, String clientSecret, - String authEndpoint, String soapEndpoint) throws ETSdkException { - ETConfiguration conf = new ETConfiguration(); - conf.set("clientId", clientId); - conf.set("clientSecret", clientSecret); - conf.set("authEndpoint", authEndpoint); - conf.set("soapEndpoint", soapEndpoint); - conf.set("useOAuth2Authentication", "true"); - ClassLoader oldCL = Thread.currentThread().getContextClassLoader(); - try { - Thread.currentThread().setContextClassLoader(DataExtensionClient.class.getClassLoader()); - return new DataExtensionClient(new ETClient(conf), dataExtensionKey); - } finally { - Thread.currentThread().setContextClassLoader(oldCL); - } - } - - public String getDataExtensionKey() { - return dataExtensionKey; - } - - /** - * Validate that the given schema is compatible with the given extension. - * - * @param schema the schema to check compatibility with - * @param collector failure collector - * @throws ETSdkException if there was an error getting the column information for the data extension - */ - public void validateSchemaCompatibility(Schema schema, FailureCollector collector) throws ETSdkException { - call(() -> { - Collection columns = getDataExtensionInfo().getColumnList(); - if (columns == null || columns.isEmpty()) { - collector.addFailure(String.format("Data extension '%s' must exist.", - dataExtensionKey), null) - .withConfigProperty(MarketingCloudConf.DATA_EXTENSION); - throw collector.getOrThrowException(); - } - Map lowercaseToOriginal = schema.getFields().stream() - .map(Schema.Field::getName) - .collect(Collectors.toMap(String::toLowerCase, f -> f)); - - for (ETDataExtensionColumn column : columns) { - String columnName = column.getName(); - String originalName = lowercaseToOriginal.get(columnName.toLowerCase()); - Schema.Field schemaField = schema.getField(originalName); - if (schemaField == null) { - if (column.getIsRequired()) { - collector.addFailure( - String.format("Data extension '%s' contains a required column '%s' of type '%s' that is not present in " + - "the input schema.", dataExtensionKey, columnName, column.getType()) - , null) - .withConfigProperty(MarketingCloudConf.DATA_EXTENSION); - } - } else { - Schema fieldSchema = schemaField.getSchema(); - if (fieldSchema.isNullable()) { - fieldSchema = fieldSchema.getNonNullable(); - } - Schema.Type schemaType = fieldSchema.getType(); - if (schemaType == Schema.Type.STRING) { - // Allow strings for anything, as the API just takes strings. - // If the value is malformed, it will fail at runtime. - continue; - } - String schemaTypeStr = fieldSchema.getLogicalType() != null ? - fieldSchema.getLogicalType().getToken() : schemaType.name().toLowerCase(); - String fieldName = schemaField.getName(); - switch (column.getType()) { - case BOOLEAN: - if (schemaType != Schema.Type.BOOLEAN) { - collector.addFailure( - String.format("Column '%s' is a boolean in data extension '%s', but is a '%s' in the input schema.", - originalName, dataExtensionKey, schemaTypeStr), - "Change the field schema to ensure it is a boolean or string type.") - .withInputSchemaField(fieldName); - } - break; - case DECIMAL: - if (fieldSchema.getLogicalType() != Schema.LogicalType.DECIMAL) { - collector.addFailure( - String.format("Column '%s' is a decimal in data extension '%s', but is a '%s' in the input schema.", - originalName, dataExtensionKey, schemaTypeStr), - "Change the field schema to ensure it is a decimal or string type.").withInputSchemaField(fieldName); - } - break; - case PHONE: - case TEXT: - case EMAIL_ADDRESS: - case LOCALE: - collector.addFailure( - String.format("Column '%s' is a %s in data extension '%s', but is a '%s' in the input schema.", - originalName, column.getType().name().toLowerCase(), dataExtensionKey, - schemaTypeStr), - "Change the field schema to ensure it is a string type.") - .withInputSchemaField(fieldName); - break; - case DATE: - if (fieldSchema.getLogicalType() != Schema.LogicalType.DATE) { - collector.addFailure( - String.format("Column '%s' is a date in data extension '%s', but is a '%s' in the input schema.", - originalName, dataExtensionKey, schemaTypeStr), - "Change the field schema to ensure it is a date or string type." - ).withInputSchemaField(fieldName); - } - break; - case NUMBER: - if (schemaType != Schema.Type.INT) { - collector.addFailure( - String.format("Column '%s' is a number in data extension '%s', but is a '%s' in the input schema.", - originalName, dataExtensionKey, schemaTypeStr), - "Change the field schema to ensure it is an integer or string type.").withInputSchemaField(fieldName); - } - break; - default: - collector.addFailure( - String.format("Unknown type '%s' for column '%s' in data extension '%s'.", - column.getType(), column.getName(), dataExtensionKey), - "Supported types are: boolean, decimal, phone, text, email_address, locale, date and number.") - .withInputSchemaField(fieldName); - } - } - } - return null; - }); - } - - public DataExtensionInfo getDataExtensionInfo() throws ETSdkException { - return call(() -> { - List columns = ETDataExtension.retrieveColumns(client, dataExtensionKey); - return new DataExtensionInfo(dataExtensionKey, columns); - }); - } - - public List scan() throws ETSdkException { - return call(() -> ETDataExtension.select(client, "key=" + dataExtensionKey).getObjects()); - } - - public ETResponse insert(List rows) throws ETSdkException { - return call(() -> create(client, rows)); - } - - public ETResponse update(List rows) throws ETSdkException { - return call(() -> update(client, rows)); - } - - public List> upsert(List rows) throws ETSdkException { - ETResponse inserts = insert(rows); - List> result = new ArrayList<>(rows.size()); - - List toUpdate = new ArrayList<>(); - for (ETResult row : inserts.getResults()) { - // super hacky to check the error message... but there is no better way - if (row.getStatus() == ETResult.Status.ERROR && row.getErrorCode() == 2 && row.getErrorMessage() != null && - row.getErrorMessage().toLowerCase().contains("primary key")) { - ETDataExtensionRow failed = row.getObject(); - ETDataExtensionRow copy = new ETDataExtensionRow(); - copy.setDataExtensionKey(failed.getDataExtensionKey()); - for (String column : failed.getColumnNames()) { - copy.setColumn(column, failed.getColumn(column)); - } - toUpdate.add(copy); - } else { - result.add(row); - } - } - - if (!toUpdate.isEmpty()) { - ETResponse updates = update(client, toUpdate); - result.addAll(updates.getResults()); - } - return result; - } - - private T call(SFMCCall callable) throws ETSdkException { - ClassLoader oldClassloader = Thread.currentThread().getContextClassLoader(); - try { - Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); - return callable.call(); - } finally { - Thread.currentThread().setContextClassLoader(oldClassloader); - } - } - - /** - * A SFMC call - * - * @param type of return object - */ - private interface SFMCCall { - - /** - * Perform a call. - */ - T call() throws ETSdkException; - } - - /** - * Copied from ETSoapObject.create(), except modified to include the error message. - */ - private static ETResponse create(ETClient client, - List objects) throws ETSdkException { - ETResponse response = new ETResponse<>(); - - if (objects == null || objects.size() == 0) { - response.setStatus(ETResult.Status.OK); - return response; - } - - // - // Get handle to the SOAP connection: - // - - ETSoapConnection connection = client.getSoapConnection(); - - // - // Automatically refresh the token if necessary: - // - - client.refreshToken(); - - // - // Perform the SOAP create: - // - String obj = ""; - - CreateRequest createRequest = new CreateRequest(); - createRequest.setOptions(new CreateOptions()); - for (T object : objects) { - object.setClient(client); - createRequest.getObjects().add(object.toInternal()); - obj += object.getClass().getSimpleName().substring(2); - } - Soap soap = connection.getSoap("create", obj); - - CreateResponse createResponse = soap.create(createRequest); - - response.setRequestId(createResponse.getRequestID()); - if (createResponse.getOverallStatus().equals("OK")) { - response.setStatus(ETResult.Status.OK); - } else if (createResponse.getOverallStatus().equals("Error")) { - response.setStatus(ETResult.Status.ERROR); - } - response.setResponseCode(createResponse.getOverallStatus()); - response.setResponseMessage(createResponse.getOverallStatus()); - for (CreateResult createResult : createResponse.getResults()) { - // - // Allocate a new (external) object: - // - - @SuppressWarnings("unchecked") - Class externalType = (Class) objects.get(0).getClass(); - - T externalObject = null; - try { - externalObject = externalType.newInstance(); - } catch (Exception ex) { - throw new ETSdkException("could not instantiate " + externalType.getName(), ex); - } - - externalObject.setClient(client); - - // - // Convert from internal representation: - // - - // not all SOAP calls return the object though some do.. - APIObject internalObject = createResult.getObject(); - if (internalObject != null) { - externalObject.fromInternal(createResult.getObject()); - } else { - // XXX populate fields from the object passed to the call? - externalObject.setId(Integer.toString(createResult.getNewID())); - } - - // - // Add result to the list of results: - // - - ETResult result = new ETResult(); - if (createResult.getStatusCode().equals("OK")) { - result.setStatus(ETResult.Status.OK); - } else if (createResult.getStatusCode().equals("Error")) { - result.setStatus(ETResult.Status.ERROR); - } - result.setResponseCode(createResult.getStatusCode()); - result.setResponseMessage(createResult.getStatusMessage()); - result.setErrorCode(createResult.getErrorCode()); - - // this is modified, cast to DataExtensionCreateResult since it has the error message - if (createResult instanceof DataExtensionCreateResult) { - DataExtensionCreateResult extCreateResult = (DataExtensionCreateResult) createResult; - result.setErrorMessage(extCreateResult.getErrorMessage()); - } - - // this is also modified, so we know the object that failed - //if (result.getResponseCode().equals("OK")) { // XXX? - result.setObject(externalObject); - //} - response.addResult(result); - } - - return response; - } - - /** - * Copy of ETSoapObject.update() except with modifications to set error message and original object. - */ - public static ETResponse update(ETClient client, - List objects) throws ETSdkException { - ETResponse response = new ETResponse<>(); - - if (objects == null || objects.size() == 0) { - response.setStatus(ETResult.Status.OK); - return response; - } - - // - // Get handle to the SOAP connection: - // - - ETSoapConnection connection = client.getSoapConnection(); - - // - // Automatically refresh the token if necessary: - // - - client.refreshToken(); - - // - // Perform the SOAP update: - // - String obj = ""; - - UpdateRequest updateRequest = new UpdateRequest(); - updateRequest.setOptions(new UpdateOptions()); - for (ETDataExtensionRow object : objects) { - object.setClient(client); - updateRequest.getObjects().add(object.toInternal()); - obj += object.getClass().getSimpleName().substring(2); - } - Soap soap = connection.getSoap("update", obj); - - UpdateResponse updateResponse = soap.update(updateRequest); - - response.setRequestId(updateResponse.getRequestID()); - if (updateResponse.getOverallStatus().equals("OK")) { - response.setStatus(ETResult.Status.OK); - } else if (updateResponse.getOverallStatus().equals("Error")) { - response.setStatus(ETResult.Status.ERROR); - } - response.setResponseCode(updateResponse.getOverallStatus()); - response.setResponseMessage(updateResponse.getOverallStatus()); - for (UpdateResult updateResult : updateResponse.getResults()) { - // - // Allocate a new (external) object: - // - - @SuppressWarnings("unchecked") - Class externalType = (Class) objects.get(0).getClass(); - - ETDataExtensionRow externalObject = null; - try { - externalObject = externalType.newInstance(); - } catch (Exception ex) { - throw new ETSdkException("could not instantiate " - + externalType.getName(), ex); - } - - externalObject.setClient(client); - - // - // Convert from internal representation: - // - - // not all SOAP calls return the object though some do.. - APIObject internalObject = updateResult.getObject(); - if (internalObject != null) { - externalObject.fromInternal(updateResult.getObject()); - } else { - // XXX populate fields from the object passed to the call? - } - - // - // Add result to the list of results: - // - - ETResult result = new ETResult<>(); - if (updateResult.getStatusCode().equals("OK")) { - result.setStatus(ETResult.Status.OK); - } else if (updateResult.getStatusCode().equals("Error")) { - result.setStatus(ETResult.Status.ERROR); - } - result.setResponseCode(updateResult.getStatusCode()); - result.setResponseMessage(updateResult.getStatusMessage()); - result.setErrorCode(updateResult.getErrorCode()); - - - // this is modified, cast to DataExtensionCreateResult since it has the error message - if (updateResult instanceof DataExtensionUpdateResult) { - DataExtensionUpdateResult extUpdateResult = (DataExtensionUpdateResult) updateResult; - result.setErrorMessage(extUpdateResult.getErrorMessage()); - } - - // this is also modified, so we know the object that failed - //if (result.getResponseCode().equals("OK")) { // XXX? - result.setObject(externalObject); - //} - - if (result.getResponseCode().equals("OK")) { // XXX? - result.setObject(externalObject); - } - response.addResult(result); - } - - return response; - } - - /** - * Used to run the client from the command line. Primarily used for testing purposes. - * Expects arguments in the form [command] [key] [properties files] where properties file is expected - * to contain lines like: - * - * clientId=... - * clientSecret=... - * authEndpoint=... - * soapEndpoint=... - */ - public static void main(String[] args) throws Exception { - String command = args[0]; - String key = args[1]; - String propertiesFile = args[2]; - ETConfiguration conf = new ETConfiguration(propertiesFile); - - DataExtensionClient client = new DataExtensionClient(new ETClient(conf), key); - - if ("describe".equals(command)) { - Collection columns = client.getDataExtensionInfo().getColumnList(); - System.out.println(columns.stream().map(c -> c.getName() + " " + c.getType()) - .collect(Collectors.joining("\n"))); - } else if ("scan".equals(command)) { - Collection columns = client.getDataExtensionInfo().getColumnList(); - List columnNames = columns.stream().map(ETDataExtensionColumn::getName) - .collect(Collectors.toList()); - System.out.println(String.join(", ", columnNames)); - for (ETDataExtensionRow row : client.scan()) { - List values = new ArrayList<>(columnNames.size()); - for (String columnName : columnNames) { - values.add(row.getColumn(columnName)); - } - System.out.println(String.join(", ", values)); - } - } - } -} diff --git a/src/main/java/io/cdap/plugin/sfmc/DataExtensionOutputFormat.java b/src/main/java/io/cdap/plugin/sfmc/DataExtensionOutputFormat.java index 34fbf72..750407a 100644 --- a/src/main/java/io/cdap/plugin/sfmc/DataExtensionOutputFormat.java +++ b/src/main/java/io/cdap/plugin/sfmc/DataExtensionOutputFormat.java @@ -18,6 +18,7 @@ import com.exacttarget.fuelsdk.ETSdkException; import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.plugin.sfmc.common.DataExtensionClient; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.JobContext; diff --git a/src/main/java/io/cdap/plugin/sfmc/DataExtensionRecordWriter.java b/src/main/java/io/cdap/plugin/sfmc/DataExtensionRecordWriter.java index 1052e1e..7f2ffca 100644 --- a/src/main/java/io/cdap/plugin/sfmc/DataExtensionRecordWriter.java +++ b/src/main/java/io/cdap/plugin/sfmc/DataExtensionRecordWriter.java @@ -21,6 +21,7 @@ import com.exacttarget.fuelsdk.ETResult; import com.exacttarget.fuelsdk.ETSdkException; import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.plugin.sfmc.common.DataExtensionClient; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.RecordWriter; import org.apache.hadoop.mapreduce.TaskAttemptContext; diff --git a/src/main/java/io/cdap/plugin/sfmc/MarketingCloudConf.java b/src/main/java/io/cdap/plugin/sfmc/MarketingCloudConf.java index 7cb9bb7..c8a29b8 100644 --- a/src/main/java/io/cdap/plugin/sfmc/MarketingCloudConf.java +++ b/src/main/java/io/cdap/plugin/sfmc/MarketingCloudConf.java @@ -23,6 +23,7 @@ import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.api.plugin.PluginConfig; import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.plugin.sfmc.common.DataExtensionClient; import java.util.Collections; import java.util.HashMap; diff --git a/src/main/java/io/cdap/plugin/sfmc/RecordDataExtensionRowConverter.java b/src/main/java/io/cdap/plugin/sfmc/RecordDataExtensionRowConverter.java index e2e905d..ef619e2 100644 --- a/src/main/java/io/cdap/plugin/sfmc/RecordDataExtensionRowConverter.java +++ b/src/main/java/io/cdap/plugin/sfmc/RecordDataExtensionRowConverter.java @@ -20,13 +20,12 @@ import com.exacttarget.fuelsdk.ETDataExtensionRow; import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sfmc.common.DataExtensionInfo; import java.math.BigDecimal; import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.function.BiFunction; -import java.util.function.Function; -import java.util.function.Predicate; /** * Converts StructuredRecords into DataExtensionRows. diff --git a/src/main/java/io/cdap/plugin/sfmc/common/DataExtensionClient.java b/src/main/java/io/cdap/plugin/sfmc/common/DataExtensionClient.java new file mode 100644 index 0000000..44fbbbf --- /dev/null +++ b/src/main/java/io/cdap/plugin/sfmc/common/DataExtensionClient.java @@ -0,0 +1,541 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.sfmc.common; + +import com.exacttarget.fuelsdk.ETClient; +import com.exacttarget.fuelsdk.ETConfiguration; +import com.exacttarget.fuelsdk.ETDataExtension; +import com.exacttarget.fuelsdk.ETDataExtensionColumn; +import com.exacttarget.fuelsdk.ETDataExtensionRow; +import com.exacttarget.fuelsdk.ETExpression; +import com.exacttarget.fuelsdk.ETFilter; +import com.exacttarget.fuelsdk.ETResponse; +import com.exacttarget.fuelsdk.ETResult; +import com.exacttarget.fuelsdk.ETSdkException; +import com.exacttarget.fuelsdk.ETSoapConnection; +import com.exacttarget.fuelsdk.ETSoapObject; +import com.exacttarget.fuelsdk.internal.APIObject; +import com.exacttarget.fuelsdk.internal.CreateOptions; +import com.exacttarget.fuelsdk.internal.CreateRequest; +import com.exacttarget.fuelsdk.internal.CreateResponse; +import com.exacttarget.fuelsdk.internal.CreateResult; +import com.exacttarget.fuelsdk.internal.DataExtensionCreateResult; +import com.exacttarget.fuelsdk.internal.DataExtensionUpdateResult; +import com.exacttarget.fuelsdk.internal.Soap; +import com.exacttarget.fuelsdk.internal.UpdateOptions; +import com.exacttarget.fuelsdk.internal.UpdateRequest; +import com.exacttarget.fuelsdk.internal.UpdateResponse; +import com.exacttarget.fuelsdk.internal.UpdateResult; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.plugin.sfmc.MarketingCloudConf; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Wrapper around an ETClient that understands objects at the level that the plugin cares about. + */ +public class DataExtensionClient { + private final ETClient client; + private final String dataExtensionKey; + + DataExtensionClient(ETClient client, String dataExtensionKey) { + this.client = client; + this.dataExtensionKey = dataExtensionKey; + } + + public static DataExtensionClient create(String dataExtensionKey, String clientId, String clientSecret, + String authEndpoint, String soapEndpoint) throws ETSdkException { + ETConfiguration conf = new ETConfiguration(); + conf.set("clientId", clientId); + conf.set("clientSecret", clientSecret); + conf.set("authEndpoint", authEndpoint); + conf.set("soapEndpoint", soapEndpoint); + conf.set("useOAuth2Authentication", "true"); + ClassLoader oldCL = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(DataExtensionClient.class.getClassLoader()); + return new DataExtensionClient(new ETClient(conf), dataExtensionKey); + } finally { + Thread.currentThread().setContextClassLoader(oldCL); + } + } + + public String getDataExtensionKey() { + return dataExtensionKey; + } + + /** + * Validate that the given schema is compatible with the given extension. + * + * @param schema the schema to check compatibility with + * @param collector failure collector + * @throws ETSdkException if there was an error getting the column information for the data extension + */ + public void validateSchemaCompatibility(Schema schema, FailureCollector collector) throws ETSdkException { + call(() -> { + Collection columns = getDataExtensionInfo().getColumnList(); + if (columns == null || columns.isEmpty()) { + collector.addFailure(String.format("Data extension '%s' must exist.", + dataExtensionKey), null) + .withConfigProperty(MarketingCloudConf.DATA_EXTENSION); + throw collector.getOrThrowException(); + } + Map lowercaseToOriginal = schema.getFields().stream() + .map(Schema.Field::getName) + .collect(Collectors.toMap(String::toLowerCase, f -> f)); + + for (ETDataExtensionColumn column : columns) { + String columnName = column.getName(); + String originalName = lowercaseToOriginal.get(columnName.toLowerCase()); + Schema.Field schemaField = schema.getField(originalName); + if (schemaField == null) { + if (column.getIsRequired()) { + collector.addFailure( + String.format("Data extension '%s' contains a required column '%s' of type '%s' that is not present in " + + "the input schema.", dataExtensionKey, columnName, column.getType()) + , null) + .withConfigProperty(MarketingCloudConf.DATA_EXTENSION); + } + } else { + Schema fieldSchema = schemaField.getSchema(); + if (fieldSchema.isNullable()) { + fieldSchema = fieldSchema.getNonNullable(); + } + Schema.Type schemaType = fieldSchema.getType(); + if (schemaType == Schema.Type.STRING) { + // Allow strings for anything, as the API just takes strings. + // If the value is malformed, it will fail at runtime. + continue; + } + String schemaTypeStr = fieldSchema.getLogicalType() != null ? + fieldSchema.getLogicalType().getToken() : schemaType.name().toLowerCase(); + String fieldName = schemaField.getName(); + switch (column.getType()) { + case BOOLEAN: + if (schemaType != Schema.Type.BOOLEAN) { + collector.addFailure( + String.format("Column '%s' is a boolean in data extension '%s', but is a '%s' in the input schema.", + originalName, dataExtensionKey, schemaTypeStr), + "Change the field schema to ensure it is a boolean or string type.") + .withInputSchemaField(fieldName); + } + break; + case DECIMAL: + if (fieldSchema.getLogicalType() != Schema.LogicalType.DECIMAL) { + collector.addFailure( + String.format("Column '%s' is a decimal in data extension '%s', but is a '%s' in the input schema.", + originalName, dataExtensionKey, schemaTypeStr), + "Change the field schema to ensure it is a decimal or string type.").withInputSchemaField(fieldName); + } + break; + case PHONE: + case TEXT: + case EMAIL_ADDRESS: + case LOCALE: + collector.addFailure( + String.format("Column '%s' is a %s in data extension '%s', but is a '%s' in the input schema.", + originalName, column.getType().name().toLowerCase(), dataExtensionKey, + schemaTypeStr), + "Change the field schema to ensure it is a string type.") + .withInputSchemaField(fieldName); + break; + case DATE: + if (fieldSchema.getLogicalType() != Schema.LogicalType.DATE) { + collector.addFailure( + String.format("Column '%s' is a date in data extension '%s', but is a '%s' in the input schema.", + originalName, dataExtensionKey, schemaTypeStr), + "Change the field schema to ensure it is a date or string type." + ).withInputSchemaField(fieldName); + } + break; + case NUMBER: + if (schemaType != Schema.Type.INT) { + collector.addFailure( + String.format("Column '%s' is a number in data extension '%s', but is a '%s' in the input schema.", + originalName, dataExtensionKey, schemaTypeStr), + "Change the field schema to ensure it is an integer or string type.").withInputSchemaField(fieldName); + } + break; + default: + collector.addFailure( + String.format("Unknown type '%s' for column '%s' in data extension '%s'.", + column.getType(), column.getName(), dataExtensionKey), + "Supported types are: boolean, decimal, phone, text, email_address, locale, date and number.") + .withInputSchemaField(fieldName); + } + } + } + return null; + }); + } + + public DataExtensionInfo getDataExtensionInfo() throws ETSdkException { + return call(() -> { + List columns = ETDataExtension.retrieveColumns(client, dataExtensionKey); + return new DataExtensionInfo(dataExtensionKey, columns); + }); + } + + public List scan() throws ETSdkException { + return call(() -> ETDataExtension.select(client, "key=" + dataExtensionKey).getObjects()); + } + + public List pagedScan(Integer page, Integer pageSize) throws ETSdkException { + return call(() -> ETDataExtension.select(client, "key=" + dataExtensionKey, page, pageSize) + .getObjects()); + } + + public ETDataExtension retrieveDataExtension(String key) throws ETSdkException { + client.refreshToken(); + ETExpression expression = new ETExpression(); + expression.setProperty("DataExtension.CustomerKey"); + expression.setOperator(ETExpression.Operator.EQUALS); + expression.addValue(key); + ETFilter filter = new ETFilter(); + filter.setExpression(expression); + ETResponse response = ETDataExtension.retrieve(client, ETDataExtension.class, (Integer) null, + (Integer) null, filter); + return response.getObject(); + } + + public Integer fetchRecordCount() throws ETSdkException { + return call(() -> ETDataExtension.select(client, "key=" + dataExtensionKey, 1, 1) + .getTotalCount()); + } + + public ETResponse insert(List rows) throws ETSdkException { + return call(() -> create(client, rows)); + } + + public ETResponse update(List rows) throws ETSdkException { + return call(() -> update(client, rows)); + } + + public List> upsert(List rows) throws ETSdkException { + ETResponse inserts = insert(rows); + List> result = new ArrayList<>(rows.size()); + + List toUpdate = new ArrayList<>(); + for (ETResult row : inserts.getResults()) { + // super hacky to check the error message... but there is no better way + if (row.getStatus() == ETResult.Status.ERROR && row.getErrorCode() == 2 && row.getErrorMessage() != null && + row.getErrorMessage().toLowerCase().contains("primary key")) { + ETDataExtensionRow failed = row.getObject(); + ETDataExtensionRow copy = new ETDataExtensionRow(); + copy.setDataExtensionKey(failed.getDataExtensionKey()); + for (String column : failed.getColumnNames()) { + copy.setColumn(column, failed.getColumn(column)); + } + toUpdate.add(copy); + } else { + result.add(row); + } + } + + if (!toUpdate.isEmpty()) { + ETResponse updates = update(client, toUpdate); + result.addAll(updates.getResults()); + } + return result; + } + + private T call(SFMCCall callable) throws ETSdkException { + ClassLoader oldClassloader = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); + return callable.call(); + } finally { + Thread.currentThread().setContextClassLoader(oldClassloader); + } + } + + /** + * A SFMC call + * + * @param type of return object + */ + private interface SFMCCall { + + /** + * Perform a call. + */ + T call() throws ETSdkException; + } + + /** + * Copied from ETSoapObject.create(), except modified to include the error message. + */ + private static ETResponse create(ETClient client, + List objects) throws ETSdkException { + ETResponse response = new ETResponse<>(); + + if (objects == null || objects.size() == 0) { + response.setStatus(ETResult.Status.OK); + return response; + } + + // + // Get handle to the SOAP connection: + // + + ETSoapConnection connection = client.getSoapConnection(); + + // + // Automatically refresh the token if necessary: + // + + client.refreshToken(); + + // + // Perform the SOAP create: + // + String obj = ""; + + CreateRequest createRequest = new CreateRequest(); + createRequest.setOptions(new CreateOptions()); + for (T object : objects) { + object.setClient(client); + createRequest.getObjects().add(object.toInternal()); + obj += object.getClass().getSimpleName().substring(2); + } + Soap soap = connection.getSoap("create", obj); + + CreateResponse createResponse = soap.create(createRequest); + + response.setRequestId(createResponse.getRequestID()); + if (createResponse.getOverallStatus().equals("OK")) { + response.setStatus(ETResult.Status.OK); + } else if (createResponse.getOverallStatus().equals("Error")) { + response.setStatus(ETResult.Status.ERROR); + } + response.setResponseCode(createResponse.getOverallStatus()); + response.setResponseMessage(createResponse.getOverallStatus()); + for (CreateResult createResult : createResponse.getResults()) { + // + // Allocate a new (external) object: + // + + @SuppressWarnings("unchecked") + Class externalType = (Class) objects.get(0).getClass(); + + T externalObject = null; + try { + externalObject = externalType.newInstance(); + } catch (Exception ex) { + throw new ETSdkException("could not instantiate " + externalType.getName(), ex); + } + + externalObject.setClient(client); + + // + // Convert from internal representation: + // + + // not all SOAP calls return the object though some do.. + APIObject internalObject = createResult.getObject(); + if (internalObject != null) { + externalObject.fromInternal(createResult.getObject()); + } else { + // XXX populate fields from the object passed to the call? + externalObject.setId(Integer.toString(createResult.getNewID())); + } + + // + // Add result to the list of results: + // + + ETResult result = new ETResult(); + if (createResult.getStatusCode().equals("OK")) { + result.setStatus(ETResult.Status.OK); + } else if (createResult.getStatusCode().equals("Error")) { + result.setStatus(ETResult.Status.ERROR); + } + result.setResponseCode(createResult.getStatusCode()); + result.setResponseMessage(createResult.getStatusMessage()); + result.setErrorCode(createResult.getErrorCode()); + + // this is modified, cast to DataExtensionCreateResult since it has the error message + if (createResult instanceof DataExtensionCreateResult) { + DataExtensionCreateResult extCreateResult = (DataExtensionCreateResult) createResult; + result.setErrorMessage(extCreateResult.getErrorMessage()); + } + + // this is also modified, so we know the object that failed + //if (result.getResponseCode().equals("OK")) { // XXX? + result.setObject(externalObject); + //} + response.addResult(result); + } + + return response; + } + + /** + * Copy of ETSoapObject.update() except with modifications to set error message and original object. + */ + public static ETResponse update(ETClient client, + List objects) throws ETSdkException { + ETResponse response = new ETResponse<>(); + + if (objects == null || objects.size() == 0) { + response.setStatus(ETResult.Status.OK); + return response; + } + + // + // Get handle to the SOAP connection: + // + + ETSoapConnection connection = client.getSoapConnection(); + + // + // Automatically refresh the token if necessary: + // + + client.refreshToken(); + + // + // Perform the SOAP update: + // + String obj = ""; + + UpdateRequest updateRequest = new UpdateRequest(); + updateRequest.setOptions(new UpdateOptions()); + for (ETDataExtensionRow object : objects) { + object.setClient(client); + updateRequest.getObjects().add(object.toInternal()); + obj += object.getClass().getSimpleName().substring(2); + } + Soap soap = connection.getSoap("update", obj); + + UpdateResponse updateResponse = soap.update(updateRequest); + + response.setRequestId(updateResponse.getRequestID()); + if (updateResponse.getOverallStatus().equals("OK")) { + response.setStatus(ETResult.Status.OK); + } else if (updateResponse.getOverallStatus().equals("Error")) { + response.setStatus(ETResult.Status.ERROR); + } + response.setResponseCode(updateResponse.getOverallStatus()); + response.setResponseMessage(updateResponse.getOverallStatus()); + for (UpdateResult updateResult : updateResponse.getResults()) { + // + // Allocate a new (external) object: + // + + @SuppressWarnings("unchecked") + Class externalType = (Class) objects.get(0).getClass(); + + ETDataExtensionRow externalObject = null; + try { + externalObject = externalType.newInstance(); + } catch (Exception ex) { + throw new ETSdkException("could not instantiate " + + externalType.getName(), ex); + } + + externalObject.setClient(client); + + // + // Convert from internal representation: + // + + // not all SOAP calls return the object though some do.. + APIObject internalObject = updateResult.getObject(); + if (internalObject != null) { + externalObject.fromInternal(updateResult.getObject()); + } else { + // XXX populate fields from the object passed to the call? + } + + // + // Add result to the list of results: + // + + ETResult result = new ETResult<>(); + if (updateResult.getStatusCode().equals("OK")) { + result.setStatus(ETResult.Status.OK); + } else if (updateResult.getStatusCode().equals("Error")) { + result.setStatus(ETResult.Status.ERROR); + } + result.setResponseCode(updateResult.getStatusCode()); + result.setResponseMessage(updateResult.getStatusMessage()); + result.setErrorCode(updateResult.getErrorCode()); + + + // this is modified, cast to DataExtensionCreateResult since it has the error message + if (updateResult instanceof DataExtensionUpdateResult) { + DataExtensionUpdateResult extUpdateResult = (DataExtensionUpdateResult) updateResult; + result.setErrorMessage(extUpdateResult.getErrorMessage()); + } + + // this is also modified, so we know the object that failed + //if (result.getResponseCode().equals("OK")) { // XXX? + result.setObject(externalObject); + //} + + if (result.getResponseCode().equals("OK")) { // XXX? + result.setObject(externalObject); + } + response.addResult(result); + } + + return response; + } + + /** + * Used to run the client from the command line. Primarily used for testing purposes. + * Expects arguments in the form [command] [key] [properties files] where properties file is expected + * to contain lines like: + *

+ * clientId=... + * clientSecret=... + * authEndpoint=... + * soapEndpoint=... + */ + public static void main(String[] args) throws Exception { + String command = args[0]; + String key = args[1]; + String propertiesFile = args[2]; + ETConfiguration conf = new ETConfiguration(propertiesFile); + + DataExtensionClient client = new DataExtensionClient(new ETClient(conf), key); + + if ("describe".equals(command)) { + Collection columns = client.getDataExtensionInfo().getColumnList(); + System.out.println(columns.stream().map(c -> c.getName() + " " + c.getType()) + .collect(Collectors.joining("\n"))); + } else if ("scan".equals(command)) { + Collection columns = client.getDataExtensionInfo().getColumnList(); + List columnNames = columns.stream().map(ETDataExtensionColumn::getName) + .collect(Collectors.toList()); + System.out.println(String.join(", ", columnNames)); + for (ETDataExtensionRow row : client.scan()) { + List values = new ArrayList<>(columnNames.size()); + for (String columnName : columnNames) { + values.add(row.getColumn(columnName)); + } + System.out.println(String.join(", ", values)); + } + } + } +} diff --git a/src/main/java/io/cdap/plugin/sfmc/DataExtensionInfo.java b/src/main/java/io/cdap/plugin/sfmc/common/DataExtensionInfo.java similarity index 97% rename from src/main/java/io/cdap/plugin/sfmc/DataExtensionInfo.java rename to src/main/java/io/cdap/plugin/sfmc/common/DataExtensionInfo.java index 95f6e85..277554a 100644 --- a/src/main/java/io/cdap/plugin/sfmc/DataExtensionInfo.java +++ b/src/main/java/io/cdap/plugin/sfmc/common/DataExtensionInfo.java @@ -14,7 +14,7 @@ * the License. */ -package io.cdap.plugin.sfmc; +package io.cdap.plugin.sfmc.common; import com.exacttarget.fuelsdk.ETDataExtensionColumn; diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java index 982567f..55db4c3 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java @@ -16,8 +16,11 @@ package io.cdap.plugin.sfmc.source; +import com.exacttarget.fuelsdk.ETDataExtension; import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sfmc.common.DataExtensionClient; +import io.cdap.plugin.sfmc.common.DataExtensionInfo; import io.cdap.plugin.sfmc.source.apiclient.SalesforceTableAPIClientImpl; import io.cdap.plugin.sfmc.source.apiclient.SalesforceTableDataResponse; @@ -25,6 +28,7 @@ import io.cdap.plugin.sfmc.source.util.SalesforceObjectInfo; import io.cdap.plugin.sfmc.source.util.SchemaBuilder; import io.cdap.plugin.sfmc.source.util.SourceObjectMode; +import io.cdap.plugin.sfmc.source.util.Util; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.InputFormat; @@ -39,8 +43,9 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; -import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PAGE_SIZE; +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.MAX_PAGE_SIZE; /** * Salesforce input format @@ -72,8 +77,8 @@ public static List setInput(Configuration jobConfig, Sourc private static List fetchTableInfo(SourceObjectMode mode, SalesforceSourceConfig conf) { //When mode = Table, fetch details from the table name provided in plugin config - if (mode == SourceObjectMode.SINGLEOBJECT) { - SalesforceObjectInfo tableInfo = getTableMetaData(conf.getObjectName(), conf); + if (mode == SourceObjectMode.SINGLE_OBJECT) { + SalesforceObjectInfo tableInfo = getTableMetaData(conf.getDataExtensionKey(), conf); return (tableInfo == null) ? Collections.emptyList() : Collections.singletonList(tableInfo); } @@ -81,7 +86,7 @@ private static List fetchTableInfo(SourceObjectMode mode, //and then fetch details from each of the tables. List tableInfos = new ArrayList<>(); - List tableNames = conf.getApplicationName().getTableNames(); + List tableNames = Util.splitToList(conf.getDataExtensionKeys(), ','); for (String tableName : tableNames) { SalesforceObjectInfo tableInfo = getTableMetaData(tableName, conf); if (tableInfo == null) { @@ -93,26 +98,46 @@ private static List fetchTableInfo(SourceObjectMode mode, return tableInfos; } - private static SalesforceObjectInfo getTableMetaData(String tableName, SalesforceSourceConfig conf) { + private static SalesforceObjectInfo getTableMetaData(String tableKey, SalesforceSourceConfig conf) { //Call API to fetch first record from the table - SalesforceTableAPIClientImpl restApi = new SalesforceTableAPIClientImpl(conf); - SalesforceTableDataResponse response = null; + //SalesforceTableAPIClientImpl restApi = new SalesforceTableAPIClientImpl(conf); + //SalesforceTableDataResponse response = null; //SalesforceTableDataResponse response = restApi.fetchTableSchema(tableName, // conf.getStartDate(), conf.getEndDate(), // true); - if (response == null) { - return null; - } - List columns = response.getColumns(); - if (columns == null || columns.isEmpty()) { + String tableName = ""; + + try { + DataExtensionClient client = DataExtensionClient.create(tableKey, conf.getClientId(), + conf.getClientSecret(), conf.getAuthEndpoint(), conf.getSoapEndpoint()); + + ETDataExtension dataExtension = client.retrieveDataExtension(tableKey); + if (dataExtension == null) { + return null; + } + + tableName = dataExtension.getName(); + DataExtensionInfo metaData = client.getDataExtensionInfo(); + + List columns = metaData.getColumnList().stream() + .map(o->new SalesforceColumn(o.getName(), o.getType().name())) + .collect(Collectors.toList()); + + if (columns == null || columns.isEmpty()) { + return null; + } + + SchemaBuilder schemaBuilder = new SchemaBuilder(); + Schema schema = schemaBuilder.constructSchema(tableName, columns); + + int totalRecords = client.fetchRecordCount(); + + LOG.debug("table {}, rows = {}", tableName, totalRecords); + return new SalesforceObjectInfo(tableKey, tableName, schema, totalRecords); + } catch (Exception e) { return null; } - - SchemaBuilder schemaBuilder = new SchemaBuilder(); - Schema schema = schemaBuilder.constructSchema(tableName, columns); - LOG.debug("table {}, rows = {}", tableName, response.getTotalRecordCount()); - return new SalesforceObjectInfo(tableName, schema, response.getTotalRecordCount()); } @Override @@ -120,29 +145,31 @@ public List getSplits(JobContext jobContext) throws IOException, Int SalesforceJobConfiguration jobConfig = new SalesforceJobConfiguration(jobContext.getConfiguration()); SalesforceSourceConfig pluginConf = jobConfig.getPluginConf(); + int pageSize = pluginConf.getPageSize(); List tableInfos = jobConfig.getTableInfos(); List resultSplits = new ArrayList<>(); for (SalesforceObjectInfo tableInfo : tableInfos) { + String tableKey = tableInfo.getTableKey(); String tableName = tableInfo.getTableName(); int totalRecords = tableInfo.getRecordCount(); - if (totalRecords < PAGE_SIZE) { + if (totalRecords < pageSize) { //add single split for table and continue - resultSplits.add(new SalesforceInputSplit(tableName, 0, totalRecords)); + resultSplits.add(new SalesforceInputSplit(tableKey, tableName, 0, totalRecords)); continue; } - int pages = (tableInfo.getRecordCount() / PAGE_SIZE) + 1; + int pages = (tableInfo.getRecordCount() / pageSize) + 1; int offset = 0; - int recordsOnPage = PAGE_SIZE; + int recordsOnPage = pageSize; for (int page = 1; page <= pages; page++) { if (page == pages) { recordsOnPage = totalRecords - offset; } - resultSplits.add(new SalesforceInputSplit(tableName, offset, recordsOnPage)); - offset += PAGE_SIZE; - recordsOnPage -= PAGE_SIZE; + resultSplits.add(new SalesforceInputSplit(tableKey, tableName, offset, recordsOnPage)); + offset += pageSize; + recordsOnPage -= pageSize; } } diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputSplit.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputSplit.java index b1e2749..89af820 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputSplit.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputSplit.java @@ -27,6 +27,7 @@ * Salesforce input split */ public class SalesforceInputSplit extends InputSplit implements Writable { + private String tableKey; private String tableName; private int offset; private int length; @@ -35,12 +36,17 @@ public class SalesforceInputSplit extends InputSplit implements Writable { public SalesforceInputSplit() { } - public SalesforceInputSplit(String tableName, int offset, int length) { + public SalesforceInputSplit(String tableKey, String tableName, int offset, int length) { + this.tableKey = tableKey; this.tableName = tableName; this.offset = offset; this.length = length; } + public String getTableKey() { + return tableKey; + } + public String getTableName() { return tableName; } diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java index 6be0b32..a1bf0df 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java @@ -16,12 +16,16 @@ package io.cdap.plugin.sfmc.source; +import com.exacttarget.fuelsdk.ETDataExtensionRow; import com.google.common.base.Strings; import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sfmc.common.DataExtensionClient; +import io.cdap.plugin.sfmc.common.DataExtensionInfo; import io.cdap.plugin.sfmc.source.apiclient.SalesforceTableAPIClientImpl; import io.cdap.plugin.sfmc.source.apiclient.SalesforceTableDataResponse; +import io.cdap.plugin.sfmc.source.util.SalesforceColumn; import io.cdap.plugin.sfmc.source.util.SchemaBuilder; import io.cdap.plugin.sfmc.source.util.SourceObjectMode; import org.apache.hadoop.io.NullWritable; @@ -35,12 +39,12 @@ import java.sql.SQLException; import java.util.ArrayList; +import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; - -import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PAGE_SIZE; +import java.util.stream.Collectors; /** * Record reader that reads the entire contents of a Salesforce table. @@ -53,11 +57,12 @@ public class SalesforceRecordReader extends RecordReader tableFields; private Schema schema; + private String tableKey; private String tableName; - - private List> results; - private Iterator> iterator; - private Map row; + private String tableNameField; + private List results; + private Iterator iterator; + private ETDataExtensionRow row; SalesforceRecordReader(SalesforceSourceConfig pluginConf) { this.pluginConf = pluginConf; @@ -99,8 +104,8 @@ public NullWritable getCurrentKey() { public StructuredRecord getCurrentValue() throws IOException { StructuredRecord.Builder recordBuilder = StructuredRecord.builder(schema); - if (pluginConf.getQueryMode() == SourceObjectMode.MULTIOBJECT) { - + if (pluginConf.getQueryMode() == SourceObjectMode.MULTI_OBJECT) { + recordBuilder.set(tableNameField, tableName); } try { @@ -131,48 +136,74 @@ public void close() throws IOException { } private void fetchData() { + tableKey = split.getTableKey(); tableName = split.getTableName(); + tableNameField = pluginConf.getTableNameField(); - - SalesforceTableAPIClientImpl restApi = new SalesforceTableAPIClientImpl(pluginConf); + //SalesforceTableAPIClientImpl restApi = new SalesforceTableAPIClientImpl(pluginConf); //Get the table data // results = restApi.fetchTableRecords(tableName, pluginConf.getStartDate(), pluginConf.getEndDate(), // split.getOffset(), PAGE_SIZE); - LOG.debug("size={}", results.size()); - if (!results.isEmpty()) { - fetchSchema(restApi); + try { + DataExtensionClient client = DataExtensionClient.create(tableKey, pluginConf.getClientId(), + pluginConf.getClientSecret(), pluginConf.getAuthEndpoint(), pluginConf.getSoapEndpoint()); + + results = client.pagedScan(split.getOffset(), pluginConf.getPageSize()); + + LOG.debug("size={}", results.size()); + if (!results.isEmpty()) { + fetchSchema(client); + } + + } catch (Exception e) { + results = Collections.emptyList(); } iterator = results.iterator(); } - private void fetchSchema(SalesforceTableAPIClientImpl restApi) { + private void fetchSchema(DataExtensionClient client) { //Fetch the column definition // SalesforceTableDataResponse response = restApi.fetchTableSchema(tableName, null, null, // false); + /* SalesforceTableDataResponse response = null; if (response == null) { return; } + */ + + List schemaFields; + + try { + DataExtensionInfo metaData = client.getDataExtensionInfo(); + + List columns = metaData.getColumnList().stream() + .map(o->new SalesforceColumn(o.getName(), o.getType().name())) + .collect(Collectors.toList()); - //Build schema - SchemaBuilder schemaBuilder = new SchemaBuilder(); - Schema tempSchema = schemaBuilder.constructSchema(tableName, response.getColumns()); - tableFields = tempSchema.getFields(); - List schemaFields = new ArrayList<>(tableFields); + //Build schema + SchemaBuilder schemaBuilder = new SchemaBuilder(); + Schema tempSchema = schemaBuilder.constructSchema(tableName, columns); + tableFields = tempSchema.getFields(); + schemaFields = new ArrayList<>(tableFields); - if (pluginConf.getQueryMode() == SourceObjectMode.MULTIOBJECT) { + if (pluginConf.getQueryMode() == SourceObjectMode.MULTI_OBJECT) { + schemaFields.add(Schema.Field.of(tableNameField, Schema.of(Schema.Type.STRING))); + } + } catch (Exception e) { + schemaFields = Collections.emptyList(); } schema = Schema.recordOf(tableName, schemaFields); } - private Object convertToValue(String fieldName, Schema fieldSchema, Map record) { + private Object convertToValue(String fieldName, Schema fieldSchema, ETDataExtensionRow record) { Schema.Type fieldType = fieldSchema.getType(); - Object fieldValue = record.get(fieldName); + Object fieldValue = record.getColumn(fieldName); switch (fieldType) { case STRING: @@ -211,7 +242,7 @@ private Object convertToValue(String fieldName, Schema fieldSchema, Map nestedRecord = (Map) fieldValue; - StructuredRecord.Builder nestedRecordBuilder = StructuredRecord.builder(fieldSchema); - Objects.requireNonNull(fieldSchema.getFields(), "Nested Schema fields cannot be empty").forEach( - nestedField -> { - String nestedFieldName = nestedField.getName(); - Object nestedFieldValue = convertToValue(nestedFieldName, nestedField.getSchema(), nestedRecord); - nestedRecordBuilder.set(nestedFieldName, nestedFieldValue); - } - ); - return nestedRecordBuilder.build(); - } else { - throw new IllegalStateException( - String.format("Field '%s' is of unexpected type '%s'. Declared 'RECORD' types: %s", - fieldName, fieldValue.getClass().getSimpleName(), fieldSchema.toString())); - } - } } diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java index fd2524a..5b4fcdf 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java @@ -19,18 +19,19 @@ import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.annotation.Plugin; - +import io.cdap.cdap.api.data.batch.Input; import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.api.dataset.lib.KeyValue; import io.cdap.cdap.etl.api.Emitter; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.PipelineConfigurer; import io.cdap.cdap.etl.api.StageConfigurer; - +import io.cdap.cdap.etl.api.action.SettableArguments; import io.cdap.cdap.etl.api.batch.BatchSource; import io.cdap.cdap.etl.api.batch.BatchSourceContext; - - +import io.cdap.plugin.common.LineageRecorder; +import io.cdap.plugin.common.SourceInputFormatProvider; import io.cdap.plugin.sfmc.source.util.SalesforceObjectInfo; import io.cdap.plugin.sfmc.source.util.SourceObjectMode; import org.apache.hadoop.conf.Configuration; @@ -39,8 +40,12 @@ import org.slf4j.LoggerFactory; import java.util.Collection; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PLUGIN_NAME; +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.TABLE_PREFIX; /** @@ -84,17 +89,32 @@ public void prepareRun(BatchSourceContext context) throws Exception { Configuration hConf = new Configuration(); Collection tables = SalesforceInputFormat.setInput(hConf, mode, conf); - /* SettableArguments arguments = context.getArguments(); + SettableArguments arguments = context.getArguments(); for (SalesforceObjectInfo tableInfo : tables) { arguments.set(TABLE_PREFIX + tableInfo.getTableName(), tableInfo.getSchema().toString()); + recordLineage(context, tableInfo); } context.setInput(Input.of(conf.getReferenceName(), - new SourceInputFormatProvider(ServiceNowInputFormat.class, hConf)));*/ + new SourceInputFormatProvider(SalesforceInputFormat.class, hConf))); } @Override public void transform(KeyValue input, Emitter emitter) { emitter.emit(input.getValue()); } + + private void recordLineage(BatchSourceContext context, SalesforceObjectInfo tableInfo) { + String tableName = tableInfo.getTableName(); + String outputName = String.format("%s-%s", conf.getReferenceName(), tableName); + Schema schema = tableInfo.getSchema(); + LineageRecorder lineageRecorder = new LineageRecorder(context, outputName); + lineageRecorder.createExternalDataset(schema); + List fields = Objects.requireNonNull(schema).getFields(); + if (fields != null && !fields.isEmpty()) { + lineageRecorder.recordRead("Read", + String.format("Read from '%s' Salesforce table.", tableName), + fields.stream().map(Schema.Field::getName).collect(Collectors.toList())); + } + } } diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java index 7823ceb..fa10412 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java @@ -16,46 +16,41 @@ package io.cdap.plugin.sfmc.source; +import com.exacttarget.fuelsdk.ETSdkException; import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Macro; import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.plugin.PluginConfig; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.plugin.common.Constants; -import io.cdap.plugin.sfmc.source.apiclient.SalesforceTableAPIClientImpl; -import io.cdap.plugin.sfmc.source.util.SourceApplication; +import io.cdap.plugin.sfmc.common.DataExtensionClient; import io.cdap.plugin.sfmc.source.util.SourceObjectMode; -import io.cdap.plugin.sfmc.source.util.SourceValueType; import io.cdap.plugin.sfmc.source.util.Util; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - -import java.time.LocalDate; -import java.time.temporal.ChronoUnit; +import java.util.List; import java.util.Optional; import javax.annotation.Nullable; -import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.DATE_FORMAT; +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.MAX_PAGE_SIZE; import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_API_ENDPOINT; import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_AUTH_API_ENDPOINT; import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_CLIENT_ID; import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_CLIENT_SECRET; -import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_END_DATE; -import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_MULTIOBJECT_NAME; -import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_OBJECT_NAME; +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_DATA_EXTENSION_KEY; +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_DATA_EXTENSION_KEY_LIST; +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_PAGE_SIZE; import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_QUERY_MODE; import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_SOAP_API_ENDPOINT; -import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_START_DATE; - - +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_TABLE_NAME_FIELD; -import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_VALUE_TYPE; /** * Configuration for the {@link SalesforceSource}. */ public class SalesforceSourceConfig extends PluginConfig { private static final Logger LOG = LoggerFactory.getLogger(SalesforceSourceConfig.class); + @Name(Constants.Reference.REFERENCE_NAME) @Description("This will be used to uniquely identify this source for lineage, annotating metadata, etc.") private String referenceName; @@ -67,7 +62,7 @@ public class SalesforceSourceConfig extends PluginConfig { + "`Table` - will allow user to enter table name for which data will be fetched.") private String queryMode; - @Name(PROPERTY_MULTIOBJECT_NAME) + @Name(PROPERTY_DATA_EXTENSION_KEY_LIST) @Macro @Nullable @Description("Application name for which data to be fetched. The application can be one of three values: " + @@ -75,14 +70,22 @@ public class SalesforceSourceConfig extends PluginConfig { "`Product Catalog` - will fetch data for all tables under Product Catalog application, " + "`Procurement` - will fetch data for all tables under Procurement application. " + "Note, the Application name value will be ignored if the Mode is set to `Table`.") - private String multiObjectName; + private String dataExtensionKeys; + + @Name(PROPERTY_TABLE_NAME_FIELD) + @Macro + @Nullable + @Description("The name of the field that holds the table name. Must not be the name of any table column that " + + "will be read. Defaults to `tablename`. Note, the Table name field value will be ignored if the Mode " + + "is set to `Table`.") + private String tableNameField; - @Name(PROPERTY_OBJECT_NAME) + @Name(PROPERTY_DATA_EXTENSION_KEY) @Macro @Nullable @Description("The name of the Salesforce table from which data to be fetched. Note, the Table name value " + "will be ignored if the Mode is set to `Reporting`.") - private String objectName; + private String dataExtensionKey; @Name(PROPERTY_CLIENT_ID) @Macro @@ -102,52 +105,34 @@ public class SalesforceSourceConfig extends PluginConfig { @Name(PROPERTY_AUTH_API_ENDPOINT) @Macro @Description("The AUTH API Endpoint for Salesforce Instance. " + - "For example, https://instance.auth.marketingcloudapis.com") + "For example, https://instance.auth.marketingcloudapis.com") private String authEndpoint; @Name(PROPERTY_SOAP_API_ENDPOINT) @Macro @Description("The SOAP API Endpoint for Salesforce Instance. " + - "For example https://instance.Salesforce.soap.marketingcloudapis.com/Service.asmx") + "For example https://instance.Salesforce.soap.marketingcloudapis.com/Service.asmx") private String soapEndpoint; - @Name(PROPERTY_VALUE_TYPE) - @Macro - @Nullable - @Description("The type of values to be returned. The type can be one of two values: " - + "`Actual` - will fetch the actual values from the Salesforce tables, " - + "`Display` - will fetch the display values from the Salesforce tables.") - private String valueType; - - @Name(PROPERTY_START_DATE) - @Macro - @Nullable - @Description("The Start date to be used to filter the data. The format must be 'yyyy-MM-dd'.") - private String startDate; - - @Name(PROPERTY_END_DATE) + @Name(PROPERTY_PAGE_SIZE) @Macro - @Nullable - @Description("The End date to be used to filter the data. The format must be 'yyyy-MM-dd'.") - private String endDate; + @Description("Maximum number of records that can be read from Salesforce Marketing Cloud in one page. " + + "The minimum value is 1 and maximum value is 2500") + private int pageSize; - public SalesforceSourceConfig(String referenceName, String queryMode, @Nullable String multiObjectName, - @Nullable String objectName, String clientId, - String clientSecret, String restEndpoint, String authEndpoint, String soapEndpoint, - @Nullable String valueType, @Nullable String startDate, @Nullable String endDate) { + public SalesforceSourceConfig(String referenceName, String queryMode, @Nullable String dataExtensionKeys, + @Nullable String dataExtensionKey, String clientId, String clientSecret, + String restEndpoint, String authEndpoint, String soapEndpoint, int pageSize) { this.referenceName = referenceName; this.queryMode = queryMode; - this.multiObjectName = multiObjectName; - - this.objectName = objectName; + this.dataExtensionKeys = dataExtensionKeys; + this.dataExtensionKey = dataExtensionKey; this.clientId = clientId; this.clientSecret = clientSecret; this.restEndpoint = restEndpoint; this.authEndpoint = authEndpoint; this.soapEndpoint = soapEndpoint; - this.valueType = valueType; - this.startDate = startDate; - this.endDate = endDate; + this.pageSize = pageSize; } public String getReferenceName() { @@ -172,30 +157,19 @@ public SourceObjectMode getQueryMode() { return sourceQueryMode.isPresent() ? sourceQueryMode.get() : null; } - public SourceApplication getApplicationName(FailureCollector collector) { - SourceApplication application = getApplicationName(); - if (application != null) { - return application; - } - - collector.addFailure("Unsupported multiObject name value: " + multiObjectName, - String.format("Supported applications are: %s", SourceApplication.getSupportedApplications())) - .withConfigProperty(PROPERTY_MULTIOBJECT_NAME); - throw collector.getOrThrowException(); + @Nullable + public String getDataExtensionKeys() { + return dataExtensionKeys; } @Nullable - public SourceApplication getApplicationName() { - Optional sourceApplication = SourceApplication.fromValue(multiObjectName); - - return sourceApplication.isPresent() ? sourceApplication.get() : null; + public String getTableNameField() { + return tableNameField; } - - @Nullable - public String getObjectName() { - return objectName; + public String getDataExtensionKey() { + return dataExtensionKey; } public String getClientId() { @@ -218,33 +192,8 @@ public String getSoapEndpoint() { return soapEndpoint; } - public SourceValueType getValueType(FailureCollector collector) { - SourceValueType type = getValueType(); - if (type != null) { - return type; - } - - collector.addFailure("Unsupported type value: " + valueType, - String.format("Supported value types are: %s", SourceValueType.getSupportedValueTypes())) - .withConfigProperty(PROPERTY_VALUE_TYPE); - throw collector.getOrThrowException(); - } - - @Nullable - public SourceValueType getValueType() { - Optional sourceValueType = SourceValueType.fromValue(valueType); - - return sourceValueType.isPresent() ? sourceValueType.get() : null; - } - - @Nullable - public String getStartDate() { - return startDate; - } - - @Nullable - public String getEndDate() { - return endDate; + public int getPageSize() { + return pageSize; } /** @@ -256,8 +205,7 @@ public void validate(FailureCollector collector) { validateCredentials(collector); validateQueryMode(collector); - getValueType(collector); - validateDateRange(collector); + validatePageSize(collector); } private void validateCredentials(FailureCollector collector) { @@ -292,11 +240,9 @@ private void validateCredentials(FailureCollector collector) { try { LOG.info("Called SalesforceTableAPIClientImpl"); - SalesforceTableAPIClientImpl restApi = new SalesforceTableAPIClientImpl(this); + DataExtensionClient.create("", clientId, clientSecret, authEndpoint, soapEndpoint); LOG.info("Completed SalesforceTableAPIClientImpl"); - // restApi.getAccessToken(); - //1- TODO - } catch (Exception e) { + } catch (ETSdkException e) { collector.addFailure("Unable to connect to Salesforce Instance.", "Ensure properties like Client ID, Client Secret, API Endpoint, Soap Endpoint, Auth Endpoint " + "are correct.") @@ -317,63 +263,50 @@ private void validateQueryMode(FailureCollector collector) { SourceObjectMode mode = getQueryMode(collector); - if (mode == SourceObjectMode.MULTIOBJECT) { + if (mode == SourceObjectMode.MULTI_OBJECT) { validateMultiObjectQueryMode(collector); } else { - validateObjectQueryMode(collector); + validateSingleObjectQueryMode(collector); } } private void validateMultiObjectQueryMode(FailureCollector collector) { - if (!containsMacro(PROPERTY_MULTIOBJECT_NAME)) { - getApplicationName(collector); - } - - - - - } - - private void validateObjectQueryMode(FailureCollector collector) { - if (containsMacro(PROPERTY_OBJECT_NAME)) { + if (containsMacro(PROPERTY_DATA_EXTENSION_KEY_LIST) || containsMacro(PROPERTY_TABLE_NAME_FIELD)) { return; } - if (Util.isNullOrEmpty(objectName)) { - collector.addFailure("Object name must be specified.", null) - .withConfigProperty(PROPERTY_OBJECT_NAME); + List dataExtensionKeyList = Util.splitToList(getDataExtensionKeys(), ','); + if (dataExtensionKeyList.isEmpty()) { + collector.addFailure("At least 1 Data Extension Key must be specified.", null) + .withConfigProperty(PROPERTY_DATA_EXTENSION_KEY_LIST); } - } - private void validateDateRange(FailureCollector collector) { - if (containsMacro(PROPERTY_START_DATE) || containsMacro(PROPERTY_END_DATE)) { - return; + if (Util.isNullOrEmpty(tableNameField)) { + collector.addFailure("Table name field must be specified.", null) + .withConfigProperty(PROPERTY_TABLE_NAME_FIELD); } + } - if (Util.isNullOrEmpty(startDate) || Util.isNullOrEmpty(endDate)) { + private void validateSingleObjectQueryMode(FailureCollector collector) { + if (containsMacro(PROPERTY_DATA_EXTENSION_KEY)) { return; } - //validate the date formats for both start date & end date - if (!Util.isValidDateFormat(DATE_FORMAT, startDate)) { - collector.addFailure("Invalid format for Start date. Correct Format: " + DATE_FORMAT, null) - .withConfigProperty(PROPERTY_START_DATE); + if (Util.isNullOrEmpty(dataExtensionKey)) { + collector.addFailure("Data Extension Key must be specified.", null) + .withConfigProperty(PROPERTY_DATA_EXTENSION_KEY); } + } - if (!Util.isValidDateFormat(DATE_FORMAT, endDate)) { - collector.addFailure("Invalid format for End date. Correct Format:" + DATE_FORMAT, null) - .withConfigProperty(PROPERTY_END_DATE); + private void validatePageSize(FailureCollector collector) { + if (containsMacro(PROPERTY_PAGE_SIZE)) { + return; } - //validate the date range by checking if start date is smaller than end date - LocalDate fromDate = LocalDate.parse(startDate); - LocalDate toDate = LocalDate.parse(endDate); - long noOfDays = ChronoUnit.DAYS.between(fromDate, toDate); - - if (noOfDays < 0) { - collector.addFailure("End date must be greater than Start date.", null) - .withConfigProperty(PROPERTY_START_DATE) - .withConfigProperty(PROPERTY_END_DATE); + if (pageSize < 1 || pageSize > MAX_PAGE_SIZE) { + collector.addFailure(String.format("Invalid page size '%d'.", pageSize), + String.format("Ensure the page size is at least 1 or at most '%d'", MAX_PAGE_SIZE)) + .withConfigProperty(PROPERTY_PAGE_SIZE); } } diff --git a/src/main/java/io/cdap/plugin/sfmc/source/apiclient/SalesforceTableAPIClientImpl.java b/src/main/java/io/cdap/plugin/sfmc/source/apiclient/SalesforceTableAPIClientImpl.java index b4d484a..fa0bfcd 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/apiclient/SalesforceTableAPIClientImpl.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/apiclient/SalesforceTableAPIClientImpl.java @@ -23,7 +23,7 @@ import com.google.gson.JsonObject; import com.google.gson.reflect.TypeToken; -import io.cdap.plugin.sfmc.DataExtensionClient; +import io.cdap.plugin.sfmc.common.DataExtensionClient; import io.cdap.plugin.sfmc.source.SalesforceSourceConfig; import io.cdap.plugin.sfmc.source.util.SalesforceColumn; import io.cdap.plugin.sfmc.source.util.TableConnection; diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java index 82cc3d2..164ae78 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java @@ -24,7 +24,7 @@ public interface SalesforceConstants { /** * Salesforce plugin name. */ - String PLUGIN_NAME = "Salesforce"; + String PLUGIN_NAME = "SalesforceDataExtension"; /** * Configuration property name used to specify the query mode. @@ -32,19 +32,19 @@ public interface SalesforceConstants { String PROPERTY_QUERY_MODE = "queryMode"; /** - * Configuration property name used to specify the MultiObject. + * Configuration property name used to specify data extension key */ - String PROPERTY_MULTIOBJECT_NAME = "multiObjectName"; + String PROPERTY_DATA_EXTENSION_KEY = "dataExtensionKey"; /** - * Configuration property name used to specify the table name field. + * Configuration property name used to specify the comma-separated list of data extension keys. */ - String PROPERTY_TABLE_NAME_FIELD = "tableNameField"; + String PROPERTY_DATA_EXTENSION_KEY_LIST = "dataExtensionKeyList"; /** - * Configuration property name used to specify Object name + * Configuration property name used to specify the table name field. */ - String PROPERTY_OBJECT_NAME = "objectName"; + String PROPERTY_TABLE_NAME_FIELD = "tableNameField"; /** * Configuration property name used to specify client id @@ -62,32 +62,22 @@ public interface SalesforceConstants { String PROPERTY_API_ENDPOINT = "restEndpoint"; /** - * Configuration property name used to specify user name + * Configuration property name used to specify Auth API endpoint */ String PROPERTY_AUTH_API_ENDPOINT = "authEndpoint"; /** - * Configuration property name used to specify password + * Configuration property name used to specify SOAP endpoint */ String PROPERTY_SOAP_API_ENDPOINT = "soapEndpoint"; /** - * Configuration property name used to specify value type - */ - String PROPERTY_VALUE_TYPE = "valueType"; - - /** - * Configuration property name used to specify start date - */ - String PROPERTY_START_DATE = "startDate"; - - /** - * Configuration property name used to specify end date + * Configuration property name used to specify page size */ - String PROPERTY_END_DATE = "endDate"; + String PROPERTY_PAGE_SIZE = "pageSize"; String TABLE_PREFIX = "multisink."; String DATE_FORMAT = "yyyy-MM-dd"; - int PAGE_SIZE = 5000; + int MAX_PAGE_SIZE = 2500; } diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceObjectInfo.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceObjectInfo.java index 1eba8ed..e485af9 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceObjectInfo.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceObjectInfo.java @@ -22,16 +22,22 @@ * Information about a Salesforce table. */ public class SalesforceObjectInfo { + private final String tableKey; private final String tableName; private final Schema schema; private final int recordCount; - public SalesforceObjectInfo(String tableName, Schema schema, int recordCount) { + public SalesforceObjectInfo(String tableKey, String tableName, Schema schema, int recordCount) { + this.tableKey = tableKey; this.tableName = tableName; this.schema = schema; this.recordCount = recordCount; } + public String getTableKey() { + return tableKey; + } + public String getTableName() { return tableName; } diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SchemaBuilder.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SchemaBuilder.java index 3c15486..95b93ba 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SchemaBuilder.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SchemaBuilder.java @@ -61,27 +61,14 @@ private Schema createSchema(SalesforceColumn column) { switch (column.getTypeName().toLowerCase()) { case "decimal": return Schema.of(Schema.Type.DOUBLE); - case "integer": + case "number": return Schema.of(Schema.Type.INT); case "boolean": return Schema.of(Schema.Type.BOOLEAN); - case "reference": - /* - List fields = Arrays.asList( - Schema.Field.of("link", Schema.nullableOf(Schema.of(Schema.Type.STRING))), - Schema.Field.of("value", Schema.nullableOf(Schema.of(Schema.Type.STRING))) - ); - return Schema.recordOf(column.getFieldName(), fields); - */ - case "currency": - case "glide_date": - case "glide_date_time": - case "sys_class_name": - case "domain_id": - case "domain_path": - case "guid": - case "translated_html": - case "journal": + case "phone": + case "date": + case "email_address": + case "locale": case "string": default: return Schema.of(Schema.Type.STRING); diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SourceObjectMode.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SourceObjectMode.java index fd179d5..a6fd5bc 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SourceObjectMode.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SourceObjectMode.java @@ -29,12 +29,12 @@ public enum SourceObjectMode { /** * Mode will be used as Reporting for querying data */ - MULTIOBJECT("Multi-Object"), + MULTI_OBJECT("Multi Object"), /** * Mode will be used as Table for querying data */ - SINGLEOBJECT("Single-Object"); + SINGLE_OBJECT("Single Object"); private final String value; diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/TableConnection.java b/src/main/java/io/cdap/plugin/sfmc/source/util/TableConnection.java index 447932c..2f73fbd 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/TableConnection.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/TableConnection.java @@ -19,7 +19,7 @@ import com.exacttarget.fuelsdk.ETConfiguration; import com.exacttarget.fuelsdk.ETSdkException; -import io.cdap.plugin.sfmc.DataExtensionClient; +import io.cdap.plugin.sfmc.common.DataExtensionClient; import io.cdap.plugin.sfmc.source.SalesforceSourceConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/Util.java b/src/main/java/io/cdap/plugin/sfmc/source/util/Util.java index 0bda13a..8ce3f8e 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/Util.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/Util.java @@ -16,11 +16,14 @@ package io.cdap.plugin.sfmc.source.util; +import com.google.common.base.Splitter; import com.google.common.base.Strings; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.util.Collections; import java.util.Date; +import java.util.List; /** * Utility class. @@ -30,17 +33,11 @@ public static boolean isNullOrEmpty(String string) { return Strings.isNullOrEmpty(Strings.nullToEmpty(string).trim()); } - public static boolean isValidDateFormat(String dateFormat, String value) { - final SimpleDateFormat sdf = new SimpleDateFormat(dateFormat); - try { - final Date date = sdf.parse(value); - - if (!value.equals(sdf.format(date))) { - return false; - } - } catch (ParseException e) { - return false; + public static List splitToList(String value, char delimiter) { + if (isNullOrEmpty(value)) { + return Collections.emptyList(); } - return true; + + return Splitter.on(delimiter).trimResults().splitToList(value); } } diff --git a/widgets/Salesforce-batchsource.json b/widgets/SalesforceDataExtension-batchsource.json similarity index 59% rename from widgets/Salesforce-batchsource.json rename to widgets/SalesforceDataExtension-batchsource.json index 6ef7473..eb110ac 100644 --- a/widgets/Salesforce-batchsource.json +++ b/widgets/SalesforceDataExtension-batchsource.json @@ -2,7 +2,7 @@ "metadata": { "spec-version": "1.0" }, - "display-name": "Salesforce", + "display-name": "Salesforce Marketing", "configuration-groups": [ { "label": "Basic", @@ -17,59 +17,62 @@ }, { "widget-type": "select", - "label": "Data retrieval mode", + "label": "Data Retrieval Mode", "name": "queryMode", "widget-attributes": { "values": [ - "Multi-Object", - "Single-Object" + "Multi Object", + "Single Object" ], - "default": "Single-object" + "default": "Single Object" } } ] }, { - "label": "Single-Object Mode", + "label": "Single Object Retrieval", "properties": [ { - "widget-type": "select", - "label": "Object Name", - "name": "objectName", + "widget-type": "textbox", + "label": "Data Extension External Key", + "name": "dataExtensionKey", "widget-attributes": { - "values": [ - "DataExtension", - "Campaign", - "Account", - "Email" - ], - "default": "DataExtension" + "placeholder": "External key of the Data Extension from which data to be read." } } ] }, { - "label": "Multi-Object Mode", + "label": "Multi Object Retrieval", "properties": [ + { + "widget-type": "csv", + "label": "Data Extension External Keys", + "name": "dataExtensionKeyList", + "widget-attributes": { + "placeholder": "External keys of the Data Extensions from which data to be read." + } + }, { "widget-type": "textbox", - "label": "List Of Object Name", - "name": "multiObjectName", + "label": "Table Name Field", + "name": "tableNameField", "widget-attributes": { - "placeholder": "Salesforce object name from which data to be fetched" + "placeholder": "The name of the field that holds the table name.", + "default": "tablename" } } ] }, { - "label": "Credentials", + "label": "Authentication", "properties": [ { "widget-type": "textbox", "label": "Client ID", "name": "clientId", "widget-attributes" : { - "placeholder": "Salesforce Client ID" + "placeholder": "Salesforce Marketing Cloud Client ID" } }, { @@ -77,15 +80,15 @@ "label": "Client Secret", "name": "clientSecret", "widget-attributes" : { - "placeholder": "Salesforce Client Secret Key" + "placeholder": "Salesforce Marketing Cloud Client Secret" } }, { "widget-type": "textbox", - "label": "AUTH API Endpoint", + "label": "Authentication Base URI", "name": "authEndpoint", "widget-attributes" : { - "placeholder": "Salesforce REST API Endpoint e.g. https://instance.auth.marketingcloudapis.com" + "placeholder": "Salesforce Auth API Endpoint e.g. https://instance.auth.marketingcloudapis.com" } }, { @@ -98,7 +101,7 @@ }, { "widget-type": "textbox", - "label": "REST API Endpoint", + "label": "REST API Base URI", "name": "restEndpoint", "widget-attributes" : { "placeholder": "Salesforce REST API Endpoint e.g. https://instance.rest.marketingcloudapis.com" @@ -110,31 +113,14 @@ "label": "Advanced", "properties": [ { - "widget-type": "select", - "label": "Type of values", - "name": "valueType", - "widget-attributes": { - "values": [ - "Actual", - "Display" - ], - "default": "Actual" - } - }, - { - "widget-type": "textbox", - "label": "Start Date", - "name": "startDate", - "widget-attributes" : { - "placeholder": "Start date to be used to filter the data" - } - }, - { - "widget-type": "textbox", - "label": "End Date", - "name": "endDate", + "widget-type": "number", + "label": "Page Size", + "name": "pageSize", "widget-attributes" : { - "placeholder": "End date to be used to filter the data" + "default": "500", + "min": "1", + "max": "2500", + "placeholder": "The number of records to be fetched in one page." } } ] From 8e817d0e23e2465b0d6c62dec6904feb9bb6e305 Mon Sep 17 00:00:00 2001 From: amolkokane76 Date: Thu, 19 Mar 2020 22:18:55 +0530 Subject: [PATCH 07/19] Implemented data extensions source plugin --- .../sfmc/source/util/SourceApplication.java | 80 ------------------- .../sfmc/source/util/SourceValueType.java | 71 ---------------- 2 files changed, 151 deletions(-) delete mode 100644 src/main/java/io/cdap/plugin/sfmc/source/util/SourceApplication.java delete mode 100644 src/main/java/io/cdap/plugin/sfmc/source/util/SourceValueType.java diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SourceApplication.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SourceApplication.java deleted file mode 100644 index 7f9061e..0000000 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SourceApplication.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright © 2019 Cask Data, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package io.cdap.plugin.sfmc.source.util; - -import java.util.Arrays; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * Indicates application for which data to be fetched from multiple tables in Salesforce. - */ -public enum SourceApplication { - - /** - * Data from tables related to Contract Management will be fetched - */ - CONTRACT_MANAGEMENT("Contract Management", Arrays.asList("clm_m2m_contract_asset", "clm_m2m_rate_card_asset", - "clm_condition_checker", "clm_condition_check", "ast_contract", "clm_contract_history", - "clm_terms_and_conditions", "clm_m2m_contract_and_terms", "clm_m2m_contract_user")), - - /** - * Data from tables related to Product Catalog will be fetched - */ - PRODUCT_CATALOG("Product Catalog", Arrays.asList("new_call", "pc_hardware_cat_item", "pc_product_cat_item", - "pc_software_cat_item", "pc_vendor_cat_item")), - - /** - * Data from tables related to Procurement will be fetched - */ - PROCUREMENT("Procurement", Arrays.asList("proc_po", "proc_po_item", "proc_rec_slip", "proc_rec_slip_item")); - - private final String value; - private final List tableNames; - - SourceApplication(String value, List tableNames) { - this.value = value; - this.tableNames = tableNames; - } - - /** - * Converts application string value into {@link SourceApplication} enum. - * - * @param stringValue application string value - * @return source application in optional container - */ - public static Optional fromValue(String stringValue) { - return Stream.of(values()) - .filter(keyType -> keyType.value.equalsIgnoreCase(stringValue)) - .findAny(); - } - - public static String getSupportedApplications() { - return Arrays.stream(SourceApplication.values()).map(SourceApplication::getValue) - .collect(Collectors.joining(", ")); - } - - public String getValue() { - return value; - } - - public List getTableNames() { - return tableNames; - } -} diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SourceValueType.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SourceValueType.java deleted file mode 100644 index f1aec40..0000000 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SourceValueType.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright © 2019 Cask Data, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package io.cdap.plugin.sfmc.source.util; - -import java.util.Arrays; -import java.util.Optional; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -/** - * Indicates type of value to be returned from Salesforce Table API - */ -public enum SourceValueType { - - /** - * Actual values will be returned by Salesforce Table API - */ - SHOW_ACTUAL_VALUE("Actual", "false"), - - /** - * Display values will be returned by Salesforce Table API - */ - SHOW_DISPLAY_VALUE("Display", "true"); - - private final String valueType; - private final String value; - - SourceValueType(String valueType, String value) { - this.valueType = valueType; - this.value = value; - } - - /** - * Converts value type string value into {@link SourceValueType} enum. - * - * @param stringValue value type string value - * @return source value type in optional container - */ - public static Optional fromValue(String stringValue) { - return Stream.of(values()) - .filter(keyType -> keyType.valueType.equalsIgnoreCase(stringValue)) - .findAny(); - } - - public static String getSupportedValueTypes() { - return Arrays.stream(SourceValueType.values()).map(SourceValueType::getValueType) - .collect(Collectors.joining(", ")); - } - - public String getValueType() { - return valueType; - } - - public String getValue() { - return value; - } -} From 8729cade579e29308e2f3ad3529fdfbc924f8abb Mon Sep 17 00:00:00 2001 From: amolkokane76 Date: Fri, 20 Mar 2020 15:43:17 +0530 Subject: [PATCH 08/19] Implemented data extensions source plugin --- pom.xml | 3 ++- .../sfmc/common/DataExtensionClient.java | 27 +++++++++++-------- .../sfmc/source/SalesforceInputFormat.java | 20 +++++++++++--- .../sfmc/source/SalesforceInputSplit.java | 16 ++++++----- .../source/SalesforceJobConfiguration.java | 4 +-- .../sfmc/source/SalesforceRecordReader.java | 11 ++++++-- .../plugin/sfmc/source/SalesforceSource.java | 5 ++-- .../sfmc/source/SalesforceSourceConfig.java | 4 +-- 8 files changed, 60 insertions(+), 30 deletions(-) diff --git a/pom.xml b/pom.xml index 0b9e527..de426c7 100644 --- a/pom.xml +++ b/pom.xml @@ -373,7 +373,7 @@ org.apache.felix maven-bundle-plugin - 3.3.0 + 3.5.1 true @@ -394,6 +394,7 @@ + io.cdap cdap-maven-plugin diff --git a/src/main/java/io/cdap/plugin/sfmc/common/DataExtensionClient.java b/src/main/java/io/cdap/plugin/sfmc/common/DataExtensionClient.java index 44fbbbf..dc0268e 100644 --- a/src/main/java/io/cdap/plugin/sfmc/common/DataExtensionClient.java +++ b/src/main/java/io/cdap/plugin/sfmc/common/DataExtensionClient.java @@ -204,17 +204,19 @@ public List pagedScan(Integer page, Integer pageSize) throws .getObjects()); } - public ETDataExtension retrieveDataExtension(String key) throws ETSdkException { - client.refreshToken(); - ETExpression expression = new ETExpression(); - expression.setProperty("DataExtension.CustomerKey"); - expression.setOperator(ETExpression.Operator.EQUALS); - expression.addValue(key); - ETFilter filter = new ETFilter(); - filter.setExpression(expression); - ETResponse response = ETDataExtension.retrieve(client, ETDataExtension.class, (Integer) null, - (Integer) null, filter); - return response.getObject(); + public ETDataExtension retrieveDataExtension() throws ETSdkException { + //client.refreshToken(); + return call(() -> { + ETExpression expression = new ETExpression(); + expression.setProperty("DataExtension.CustomerKey"); + expression.setOperator(ETExpression.Operator.EQUALS); + expression.addValue(dataExtensionKey); + ETFilter filter = new ETFilter(); + filter.setExpression(expression); + ETResponse response = ETDataExtension.retrieve(client, ETDataExtension.class, (Integer) null, + (Integer) null, filter); + return response.getObject(); + }); } public Integer fetchRecordCount() throws ETSdkException { @@ -520,6 +522,9 @@ public static void main(String[] args) throws Exception { DataExtensionClient client = new DataExtensionClient(new ETClient(conf), key); + ETDataExtension dataExtension = client.retrieveDataExtension(); + System.out.println("Data Extension = " + dataExtension.getName()); + if ("describe".equals(command)) { Collection columns = client.getDataExtensionInfo().getColumnList(); System.out.println(columns.stream().map(c -> c.getName() + " " + c.getType()) diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java index 8f4dced..e6a2aca 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java @@ -107,7 +107,7 @@ private static SalesforceObjectInfo getTableMetaData(String tableKey, Salesforce DataExtensionClient client = DataExtensionClient.create(tableKey, conf.getClientId(), conf.getClientSecret(), conf.getAuthEndpoint(), conf.getSoapEndpoint()); - ETDataExtension dataExtension = client.retrieveDataExtension(tableKey); + ETDataExtension dataExtension = client.retrieveDataExtension(); if (dataExtension == null) { return null; } @@ -119,6 +119,8 @@ private static SalesforceObjectInfo getTableMetaData(String tableKey, Salesforce .map(o -> new SalesforceColumn(o.getName(), o.getType().name())) .collect(Collectors.toList()); + LOG.error("columns.size() = {}", columns.size()); + if (columns == null || columns.isEmpty()) { return null; } @@ -128,7 +130,7 @@ private static SalesforceObjectInfo getTableMetaData(String tableKey, Salesforce int totalRecords = client.fetchRecordCount(); - LOG.debug("table {}, rows = {}", tableName, totalRecords); + LOG.error("table {}, rows = {}", tableName, totalRecords); return new SalesforceObjectInfo(tableKey, tableName, schema, totalRecords); } catch (Exception e) { return null; @@ -137,6 +139,7 @@ private static SalesforceObjectInfo getTableMetaData(String tableKey, Salesforce @Override public List getSplits(JobContext jobContext) throws IOException, InterruptedException { + LOG.error("In getSplits()"); SalesforceJobConfiguration jobConfig = new SalesforceJobConfiguration(jobContext.getConfiguration()); SalesforceSourceConfig pluginConf = jobConfig.getPluginConf(); @@ -150,10 +153,11 @@ public List getSplits(JobContext jobContext) throws IOException, Int int totalRecords = tableInfo.getRecordCount(); if (totalRecords < pageSize) { //add single split for table and continue - resultSplits.add(new SalesforceInputSplit(tableKey, tableName, 0, totalRecords)); + resultSplits.add(new SalesforceInputSplit(tableKey, tableName, 1, totalRecords)); continue; } + /* int pages = (tableInfo.getRecordCount() / pageSize) + 1; int offset = 0; int recordsOnPage = pageSize; @@ -166,8 +170,16 @@ public List getSplits(JobContext jobContext) throws IOException, Int offset += pageSize; recordsOnPage -= pageSize; } + */ + + int pages = (tableInfo.getRecordCount() / pageSize) + 1; + for (int page = 1; page <= pages; page++) { + resultSplits.add(new SalesforceInputSplit(tableKey, tableName, page, pageSize)); + } } + LOG.error("# of split = {}", resultSplits.size()); + return resultSplits; } @@ -175,6 +187,8 @@ public List getSplits(JobContext jobContext) throws IOException, Int public RecordReader createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { + LOG.error("In createRecordReader()"); + SalesforceJobConfiguration jobConfig = new SalesforceJobConfiguration(taskAttemptContext.getConfiguration()); SalesforceSourceConfig pluginConf = jobConfig.getPluginConf(); diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputSplit.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputSplit.java index 89af820..51ec1ed 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputSplit.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputSplit.java @@ -29,17 +29,17 @@ public class SalesforceInputSplit extends InputSplit implements Writable { private String tableKey; private String tableName; - private int offset; + private int page; private int length; // used by mapreduce public SalesforceInputSplit() { } - public SalesforceInputSplit(String tableKey, String tableName, int offset, int length) { + public SalesforceInputSplit(String tableKey, String tableName, int page, int length) { this.tableKey = tableKey; this.tableName = tableName; - this.offset = offset; + this.page = page; this.length = length; } @@ -51,20 +51,22 @@ public String getTableName() { return tableName; } - public int getOffset() { - return offset; + public int getPage() { + return page; } @Override public void write(DataOutput dataOutput) throws IOException { + dataOutput.writeUTF(this.tableKey); dataOutput.writeUTF(this.tableName); - dataOutput.writeInt(this.offset); + dataOutput.writeInt(this.page); } @Override public void readFields(DataInput dataInput) throws IOException { + this.tableKey = dataInput.readUTF(); this.tableName = dataInput.readUTF(); - this.offset = dataInput.readInt(); + this.page = dataInput.readInt(); } @Override diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceJobConfiguration.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceJobConfiguration.java index 8e1b3f5..66fcbce 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceJobConfiguration.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceJobConfiguration.java @@ -28,8 +28,8 @@ * Allows to specify and access connection configuration properties of {@link Configuration}. */ public class SalesforceJobConfiguration { - private static final String PLUGIN_CONF_FIELD = "servicenow.plugin.conf"; - private static final String INFO_FIELD = "servicenow.table.info"; + private static final String PLUGIN_CONF_FIELD = "salesforce.plugin.conf"; + private static final String INFO_FIELD = "salesforce.table.info"; private static final Type PLUGIN_CONF_TYPE = new TypeToken() { }.getType(); diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java index 2ca6edb..5cccfe0 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java @@ -64,12 +64,14 @@ public class SalesforceRecordReader extends RecordReader schemaFields; diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java index 5b4fcdf..701ef02 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java @@ -105,15 +105,16 @@ public void transform(KeyValue input, Emitter fields = Objects.requireNonNull(schema).getFields(); if (fields != null && !fields.isEmpty()) { lineageRecorder.recordRead("Read", - String.format("Read from '%s' Salesforce table.", tableName), + String.format("Read from '%s (Key: %s)' Salesforce Data Extension.", tableName, tableKey), fields.stream().map(Schema.Field::getName).collect(Collectors.toList())); } } diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java index fa10412..a995d42 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java @@ -239,9 +239,9 @@ private void validateCredentials(FailureCollector collector) { } try { - LOG.info("Called SalesforceTableAPIClientImpl"); + //LOG.info("Called SalesforceTableAPIClientImpl"); DataExtensionClient.create("", clientId, clientSecret, authEndpoint, soapEndpoint); - LOG.info("Completed SalesforceTableAPIClientImpl"); + //LOG.info("Completed SalesforceTableAPIClientImpl"); } catch (ETSdkException e) { collector.addFailure("Unable to connect to Salesforce Instance.", "Ensure properties like Client ID, Client Secret, API Endpoint, Soap Endpoint, Auth Endpoint " + From bd45d6b534c208da277613bb26237b191c5ae0a8 Mon Sep 17 00:00:00 2001 From: amolkokane76 Date: Wed, 25 Mar 2020 12:21:23 +0530 Subject: [PATCH 09/19] Code cleanup --- .../sfmc/source/SalesforceInputFormat.java | 52 +++++++------------ .../sfmc/source/SalesforceRecordReader.java | 25 +++------ .../plugin/sfmc/source/SalesforceSource.java | 7 ++- .../sfmc/source/SalesforceSourceConfig.java | 33 ++---------- .../sfmc/source/util/SalesforceConstants.java | 6 --- .../SalesforceDataExtension-batchsource.json | 16 ------ 6 files changed, 33 insertions(+), 106 deletions(-) diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java index e6a2aca..c0db1df 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java @@ -42,6 +42,8 @@ import java.util.List; import java.util.stream.Collectors; +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.MAX_PAGE_SIZE; + /** * Salesforce input format */ @@ -67,6 +69,7 @@ public static List setInput(Configuration jobConfig, Sourc jobConf.setTableInfos(tableInfos); + LOG.debug("setInput::tableInfos = {}", tableInfos.size()); return tableInfos; } @@ -81,9 +84,9 @@ private static List fetchTableInfo(SourceObjectMode mode, //and then fetch details from each of the tables. List tableInfos = new ArrayList<>(); - List tableNames = Util.splitToList(conf.getDataExtensionKeys(), ','); - for (String tableName : tableNames) { - SalesforceObjectInfo tableInfo = getTableMetaData(tableName, conf); + List tableKeys = Util.splitToList(conf.getDataExtensionKeys(), ','); + for (String tableKey : tableKeys) { + SalesforceObjectInfo tableInfo = getTableMetaData(tableKey, conf); if (tableInfo == null) { continue; } @@ -95,12 +98,7 @@ private static List fetchTableInfo(SourceObjectMode mode, private static SalesforceObjectInfo getTableMetaData(String tableKey, SalesforceSourceConfig conf) { //Call API to fetch first record from the table - //SalesforceTableAPIClientImpl restApi = new SalesforceTableAPIClientImpl(conf); - //SalesforceTableDataResponse response = null; - //SalesforceTableDataResponse response = restApi.fetchTableSchema(tableName, - // conf.getStartDate(), conf.getEndDate(), - // true); - + LOG.debug("getTableMetaData::tableKey = {}", tableKey); String tableName = ""; try { @@ -119,31 +117,32 @@ private static SalesforceObjectInfo getTableMetaData(String tableKey, Salesforce .map(o -> new SalesforceColumn(o.getName(), o.getType().name())) .collect(Collectors.toList()); - LOG.error("columns.size() = {}", columns.size()); + LOG.debug("columns.size() = {}", columns.size()); if (columns == null || columns.isEmpty()) { return null; } SchemaBuilder schemaBuilder = new SchemaBuilder(); - Schema schema = schemaBuilder.constructSchema(tableName, columns); + //Schema schema = schemaBuilder.constructSchema(tableName, columns); + Schema schema = schemaBuilder.constructSchema(tableKey, columns); int totalRecords = client.fetchRecordCount(); - LOG.error("table {}, rows = {}", tableName, totalRecords); + LOG.debug("table {}, rows = {}", tableName, totalRecords); return new SalesforceObjectInfo(tableKey, tableName, schema, totalRecords); } catch (Exception e) { + LOG.error("Error in getTableMetaData()", e); return null; } } @Override public List getSplits(JobContext jobContext) throws IOException, InterruptedException { - LOG.error("In getSplits()"); + LOG.debug("In getSplits()"); SalesforceJobConfiguration jobConfig = new SalesforceJobConfiguration(jobContext.getConfiguration()); SalesforceSourceConfig pluginConf = jobConfig.getPluginConf(); - int pageSize = pluginConf.getPageSize(); List tableInfos = jobConfig.getTableInfos(); List resultSplits = new ArrayList<>(); @@ -151,34 +150,19 @@ public List getSplits(JobContext jobContext) throws IOException, Int String tableKey = tableInfo.getTableKey(); String tableName = tableInfo.getTableName(); int totalRecords = tableInfo.getRecordCount(); - if (totalRecords < pageSize) { + if (totalRecords < MAX_PAGE_SIZE) { //add single split for table and continue resultSplits.add(new SalesforceInputSplit(tableKey, tableName, 1, totalRecords)); continue; } - /* - int pages = (tableInfo.getRecordCount() / pageSize) + 1; - int offset = 0; - int recordsOnPage = pageSize; - - for (int page = 1; page <= pages; page++) { - if (page == pages) { - recordsOnPage = totalRecords - offset; - } - resultSplits.add(new SalesforceInputSplit(tableKey, tableName, offset, recordsOnPage)); - offset += pageSize; - recordsOnPage -= pageSize; - } - */ - - int pages = (tableInfo.getRecordCount() / pageSize) + 1; + int pages = (tableInfo.getRecordCount() / MAX_PAGE_SIZE) + 1; for (int page = 1; page <= pages; page++) { - resultSplits.add(new SalesforceInputSplit(tableKey, tableName, page, pageSize)); + resultSplits.add(new SalesforceInputSplit(tableKey, tableName, page, MAX_PAGE_SIZE)); } } - LOG.error("# of split = {}", resultSplits.size()); + LOG.debug("# of split = {}", resultSplits.size()); return resultSplits; } @@ -187,7 +171,7 @@ public List getSplits(JobContext jobContext) throws IOException, Int public RecordReader createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { - LOG.error("In createRecordReader()"); + LOG.debug("In createRecordReader()"); SalesforceJobConfiguration jobConfig = new SalesforceJobConfiguration(taskAttemptContext.getConfiguration()); SalesforceSourceConfig pluginConf = jobConfig.getPluginConf(); diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java index 5cccfe0..df68f68 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java @@ -40,6 +40,8 @@ import java.util.List; import java.util.stream.Collectors; +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.MAX_PAGE_SIZE; + /** * Record reader that reads the entire contents of a Salesforce table. */ @@ -71,9 +73,9 @@ public void initialize(InputSplit split, TaskAttemptContext context) { @Override public boolean nextKeyValue() throws IOException { - LOG.error("In nextKeyValue()"); try { if (results == null) { + LOG.debug("In nextKeyValue()"); fetchData(); } @@ -98,7 +100,6 @@ public NullWritable getCurrentKey() { @Override public StructuredRecord getCurrentValue() throws IOException { - LOG.error("In getCurrentValue()"); StructuredRecord.Builder recordBuilder = StructuredRecord.builder(schema); if (pluginConf.getQueryMode() == SourceObjectMode.MULTI_OBJECT) { @@ -139,23 +140,17 @@ private void fetchData() { tableNameField = pluginConf.getTableNameField(); LOG.error("In fetchData(), tableKey={}, tableName={}", tableKey, tableName); - //SalesforceTableAPIClientImpl restApi = new SalesforceTableAPIClientImpl(pluginConf); - //Get the table data - // results = restApi.fetchTableRecords(tableName, pluginConf.getStartDate(), pluginConf.getEndDate(), - // split.getOffset(), PAGE_SIZE); - try { DataExtensionClient client = DataExtensionClient.create(tableKey, pluginConf.getClientId(), pluginConf.getClientSecret(), pluginConf.getAuthEndpoint(), pluginConf.getSoapEndpoint()); - results = client.pagedScan(split.getPage(), pluginConf.getPageSize()); + results = client.pagedScan(split.getPage(), MAX_PAGE_SIZE); - LOG.error("size={}", results.size()); + LOG.debug("size={}", results.size()); if (!results.isEmpty()) { fetchSchema(client); } - } catch (Exception e) { results = Collections.emptyList(); LOG.error("Error while fetching data", e); @@ -166,15 +161,7 @@ private void fetchData() { private void fetchSchema(DataExtensionClient client) { //Fetch the column definition - // SalesforceTableDataResponse response = restApi.fetchTableSchema(tableName, null, null, - // false); - /* - SalesforceTableDataResponse response = null; - if (response == null) { - return; - } - */ - LOG.error("In fetchSchema()"); + LOG.debug("In fetchSchema()"); List schemaFields; diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java index 701ef02..7d18c64 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java @@ -47,7 +47,6 @@ import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PLUGIN_NAME; import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.TABLE_PREFIX; - /** * A {@link BatchSource} that reads data from multiple tables in Service Now. */ @@ -91,16 +90,22 @@ public void prepareRun(BatchSourceContext context) throws Exception { Collection tables = SalesforceInputFormat.setInput(hConf, mode, conf); SettableArguments arguments = context.getArguments(); for (SalesforceObjectInfo tableInfo : tables) { + LOG.debug("add lineage for %s table", tableInfo.getTableName()); arguments.set(TABLE_PREFIX + tableInfo.getTableName(), tableInfo.getSchema().toString()); recordLineage(context, tableInfo); } + LOG.debug("lineage added for %d tables", tables.size()); + context.setInput(Input.of(conf.getReferenceName(), new SourceInputFormatProvider(SalesforceInputFormat.class, hConf))); + + LOG.debug("end of prepareRun"); } @Override public void transform(KeyValue input, Emitter emitter) { + LOG.debug("In transform"); emitter.emit(input.getValue()); } diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java index a995d42..73c76d7 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java @@ -33,14 +33,12 @@ import java.util.Optional; import javax.annotation.Nullable; -import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.MAX_PAGE_SIZE; import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_API_ENDPOINT; import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_AUTH_API_ENDPOINT; import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_CLIENT_ID; import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_CLIENT_SECRET; import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_DATA_EXTENSION_KEY; import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_DATA_EXTENSION_KEY_LIST; -import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_PAGE_SIZE; import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_QUERY_MODE; import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_SOAP_API_ENDPOINT; import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_TABLE_NAME_FIELD; @@ -114,25 +112,19 @@ public class SalesforceSourceConfig extends PluginConfig { "For example https://instance.Salesforce.soap.marketingcloudapis.com/Service.asmx") private String soapEndpoint; - @Name(PROPERTY_PAGE_SIZE) - @Macro - @Description("Maximum number of records that can be read from Salesforce Marketing Cloud in one page. " - + "The minimum value is 1 and maximum value is 2500") - private int pageSize; - public SalesforceSourceConfig(String referenceName, String queryMode, @Nullable String dataExtensionKeys, - @Nullable String dataExtensionKey, String clientId, String clientSecret, - String restEndpoint, String authEndpoint, String soapEndpoint, int pageSize) { + @Nullable String tableNameField, @Nullable String dataExtensionKey, String clientId, + String clientSecret, String restEndpoint, String authEndpoint, String soapEndpoint) { this.referenceName = referenceName; this.queryMode = queryMode; this.dataExtensionKeys = dataExtensionKeys; + this.tableNameField = tableNameField; this.dataExtensionKey = dataExtensionKey; this.clientId = clientId; this.clientSecret = clientSecret; this.restEndpoint = restEndpoint; this.authEndpoint = authEndpoint; this.soapEndpoint = soapEndpoint; - this.pageSize = pageSize; } public String getReferenceName() { @@ -192,10 +184,6 @@ public String getSoapEndpoint() { return soapEndpoint; } - public int getPageSize() { - return pageSize; - } - /** * Validates {@link SalesforceSourceConfig} instance. */ @@ -205,7 +193,6 @@ public void validate(FailureCollector collector) { validateCredentials(collector); validateQueryMode(collector); - validatePageSize(collector); } private void validateCredentials(FailureCollector collector) { @@ -239,9 +226,7 @@ private void validateCredentials(FailureCollector collector) { } try { - //LOG.info("Called SalesforceTableAPIClientImpl"); DataExtensionClient.create("", clientId, clientSecret, authEndpoint, soapEndpoint); - //LOG.info("Completed SalesforceTableAPIClientImpl"); } catch (ETSdkException e) { collector.addFailure("Unable to connect to Salesforce Instance.", "Ensure properties like Client ID, Client Secret, API Endpoint, Soap Endpoint, Auth Endpoint " + @@ -298,18 +283,6 @@ private void validateSingleObjectQueryMode(FailureCollector collector) { } } - private void validatePageSize(FailureCollector collector) { - if (containsMacro(PROPERTY_PAGE_SIZE)) { - return; - } - - if (pageSize < 1 || pageSize > MAX_PAGE_SIZE) { - collector.addFailure(String.format("Invalid page size '%d'.", pageSize), - String.format("Ensure the page size is at least 1 or at most '%d'", MAX_PAGE_SIZE)) - .withConfigProperty(PROPERTY_PAGE_SIZE); - } - } - /** * Returns true if Salesforce can be connected to. */ diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java index 164ae78..d8c25d5 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java @@ -71,13 +71,7 @@ public interface SalesforceConstants { */ String PROPERTY_SOAP_API_ENDPOINT = "soapEndpoint"; - /** - * Configuration property name used to specify page size - */ - String PROPERTY_PAGE_SIZE = "pageSize"; - String TABLE_PREFIX = "multisink."; - String DATE_FORMAT = "yyyy-MM-dd"; int MAX_PAGE_SIZE = 2500; } diff --git a/widgets/SalesforceDataExtension-batchsource.json b/widgets/SalesforceDataExtension-batchsource.json index eb110ac..1753f33 100644 --- a/widgets/SalesforceDataExtension-batchsource.json +++ b/widgets/SalesforceDataExtension-batchsource.json @@ -108,22 +108,6 @@ } } ] - }, - { - "label": "Advanced", - "properties": [ - { - "widget-type": "number", - "label": "Page Size", - "name": "pageSize", - "widget-attributes" : { - "default": "500", - "min": "1", - "max": "2500", - "placeholder": "The number of records to be fetched in one page." - } - } - ] } ], "outputs": [ ], From 6a13e3f237901721079a232a9d1dccd33a8c6572 Mon Sep 17 00:00:00 2001 From: amolkokane76 Date: Wed, 25 Mar 2020 20:09:00 +0530 Subject: [PATCH 10/19] Updated README files --- README.md | 7 ++- docs/SalesforceDataExtension-batchsource.md | 48 ++++++++++++++++ pom.xml | 7 +-- .../sfmc/source/SalesforceInputFormat.java | 8 +-- .../sfmc/source/SalesforceRecordReader.java | 12 ++-- .../plugin/sfmc/source/SalesforceSource.java | 6 +- .../sfmc/source/SalesforceSourceConfig.java | 57 +++++++++---------- ...ceObjectMode.java => SourceQueryMode.java} | 14 ++--- 8 files changed, 102 insertions(+), 57 deletions(-) create mode 100644 docs/SalesforceDataExtension-batchsource.md rename src/main/java/io/cdap/plugin/sfmc/source/util/{SourceObjectMode.java => SourceQueryMode.java} (76%) diff --git a/README.md b/README.md index b274e08..75d37cf 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,10 @@ # Overview -Contains CDAP pipeline plugins for Salesforce Marketing Cloud +Contains CDAP pipeline plugins for Salesforce Marketing Cloud. Following plugins are available in this repository. + +* [Salesforce Marketing Cloud Data Extension Source](docs/SalesforceDataExtension-batchsource.md) +* [Salesforce Marketing Cloud Data Extension Sink](docs/SalesforceDataExtension-batchsink.md) # Contact @@ -14,7 +17,7 @@ CDAP User Group and Development Discussions: * [cdap-user@googlegroups.com](https://groups.google.com/d/forum/cdap-user) The *cdap-user* mailing list is primarily for users using the product to develop -applications or building plugins for appplications. You can expect questions from +applications or building plugins for applications. You can expect questions from users, release announcements, and any other discussions that we think will be helpful to the users. diff --git a/docs/SalesforceDataExtension-batchsource.md b/docs/SalesforceDataExtension-batchsource.md new file mode 100644 index 0000000..b563888 --- /dev/null +++ b/docs/SalesforceDataExtension-batchsource.md @@ -0,0 +1,48 @@ +# Salesforce Marketing Cloud Data Extension Source + +Description +----------- + +Reads from one or multiple data extensions within Salesforce Marketing Cloud depending on the mode values set for +this plugin. In case of `Multi Object` mode, the source will output a record for each row in the data extension it +reads, with each record containing an additional field that holds the name of the data extension from which the record +came from. In case of `Single Object` mode, this additional field will not be there in the output. In addition, for +each data extension that will be read, this plugin will set pipeline arguments where the key is +`multisink.[dataextensionname]` and the value is the schema of the data extension. + + +Properties +---------- + +**Reference Name**: Name used to uniquely identify this source for lineage, annotating metadata, etc. + +**Data Retrieval Mode**: Mode of data retrieval. The mode can be one of two values: + +`Multi Object` - will allow user to fetch data for multiple data extensions, + +`Single Object` - will allow user to fetch data for single data extension. + +**Data Extension External Key**: Specify the data extension key from which data to be fetched. Note, this value will +be ignored if the Mode is set to `Multi Object`. + +**Data Extension External Keys**: Specify the data extension keys from which data to be fetched; +for example: 'Key1,Key2'. + +Note, this value will be ignored if the Mode is set to `Single Object`. + +**Table Name Field**: The name of the field that holds the data extension name. Must not be the name of any data +extension column that will be read. Defaults to `tablename`. Note, the Table name field value will be ignored if the +Mode is set to `Single Object`. + +**Client ID**: OAuth2 client ID associated with an installed package in the Salesforce Marketing Cloud. + +**Client Secret**: TOAuth2 client secret associated with an installed package in the Salesforce Marketing Cloud. + +**Authentication Base URI**: Authentication Base URL associated for the Server-to-Server API integration. +For example, `https://instance.auth.marketingcloudapis.com/` + +**SOAP API Endpoint**: The SOAP Endpoint URL associated for the Server-to-Server API integration. For example, +`https://instance.soap.marketingcloudapis.com/Service.asmx` + +**REST API Base URI**: The REST API Base URL associated for the Server-to-Server API integration. For example, +`https://instance.rest.marketingcloudapis.com/` diff --git a/pom.xml b/pom.xml index de426c7..26e82fd 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ Salesforce Marketing plugins io.cdap.plugin salesforce-marketing-plugins - 1.1.0-SNAPSHOT + 1.2.0-SNAPSHOT jar @@ -68,11 +68,6 @@ hydrator-common ${cdap.plugin.version} - - io.cdap.plugin - hydrator-common - ${hydrator.version} - com.google.guava guava diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java index c0db1df..74ed0a9 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java @@ -24,7 +24,7 @@ import io.cdap.plugin.sfmc.source.util.SalesforceColumn; import io.cdap.plugin.sfmc.source.util.SalesforceObjectInfo; import io.cdap.plugin.sfmc.source.util.SchemaBuilder; -import io.cdap.plugin.sfmc.source.util.SourceObjectMode; +import io.cdap.plugin.sfmc.source.util.SourceQueryMode; import io.cdap.plugin.sfmc.source.util.Util; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.NullWritable; @@ -58,7 +58,7 @@ public class SalesforceInputFormat extends InputFormat setInput(Configuration jobConfig, SourceObjectMode mode, + public static List setInput(Configuration jobConfig, SourceQueryMode mode, SalesforceSourceConfig conf) { SalesforceJobConfiguration jobConf = new SalesforceJobConfiguration(jobConfig); jobConf.setPluginConfiguration(conf); @@ -73,9 +73,9 @@ public static List setInput(Configuration jobConfig, Sourc return tableInfos; } - private static List fetchTableInfo(SourceObjectMode mode, SalesforceSourceConfig conf) { + private static List fetchTableInfo(SourceQueryMode mode, SalesforceSourceConfig conf) { //When mode = Table, fetch details from the table name provided in plugin config - if (mode == SourceObjectMode.SINGLE_OBJECT) { + if (mode == SourceQueryMode.SINGLE_OBJECT) { SalesforceObjectInfo tableInfo = getTableMetaData(conf.getDataExtensionKey(), conf); return (tableInfo == null) ? Collections.emptyList() : Collections.singletonList(tableInfo); } diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java index df68f68..3d16ab3 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java @@ -24,7 +24,7 @@ import io.cdap.plugin.sfmc.common.DataExtensionInfo; import io.cdap.plugin.sfmc.source.util.SalesforceColumn; import io.cdap.plugin.sfmc.source.util.SchemaBuilder; -import io.cdap.plugin.sfmc.source.util.SourceObjectMode; +import io.cdap.plugin.sfmc.source.util.SourceQueryMode; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.RecordReader; @@ -66,7 +66,7 @@ public class SalesforceRecordReader extends RecordReader(tableFields); - if (pluginConf.getQueryMode() == SourceObjectMode.MULTI_OBJECT) { + if (pluginConf.getQueryMode() == SourceQueryMode.MULTI_OBJECT) { schemaFields.add(Schema.Field.of(tableNameField, Schema.of(Schema.Type.STRING))); } diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java index 7d18c64..a7ba356 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java @@ -33,7 +33,7 @@ import io.cdap.plugin.common.LineageRecorder; import io.cdap.plugin.common.SourceInputFormatProvider; import io.cdap.plugin.sfmc.source.util.SalesforceObjectInfo; -import io.cdap.plugin.sfmc.source.util.SourceObjectMode; +import io.cdap.plugin.sfmc.source.util.SourceQueryMode; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.NullWritable; import org.slf4j.Logger; @@ -84,14 +84,14 @@ public void prepareRun(BatchSourceContext context) throws Exception { conf.validate(collector); collector.getOrThrowException(); - SourceObjectMode mode = conf.getQueryMode(collector); + SourceQueryMode mode = conf.getQueryMode(collector); Configuration hConf = new Configuration(); Collection tables = SalesforceInputFormat.setInput(hConf, mode, conf); SettableArguments arguments = context.getArguments(); for (SalesforceObjectInfo tableInfo : tables) { LOG.debug("add lineage for %s table", tableInfo.getTableName()); - arguments.set(TABLE_PREFIX + tableInfo.getTableName(), tableInfo.getSchema().toString()); + arguments.set(TABLE_PREFIX + tableInfo.getTableKey(), tableInfo.getSchema().toString()); recordLineage(context, tableInfo); } diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java index 73c76d7..25aece5 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java @@ -23,8 +23,9 @@ import io.cdap.cdap.api.plugin.PluginConfig; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.plugin.common.Constants; +import io.cdap.plugin.common.IdUtils; import io.cdap.plugin.sfmc.common.DataExtensionClient; -import io.cdap.plugin.sfmc.source.util.SourceObjectMode; +import io.cdap.plugin.sfmc.source.util.SourceQueryMode; import io.cdap.plugin.sfmc.source.util.Util; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,61 +56,59 @@ public class SalesforceSourceConfig extends PluginConfig { @Name(PROPERTY_QUERY_MODE) @Macro - @Description("Mode of query. The mode can be one of two values: " - + "`Reporting` - will allow user to choose application for which data will be fetched for all tables, " - + "`Table` - will allow user to enter table name for which data will be fetched.") + @Description("Mode of data retrieval. The mode can be one of two values: " + + "`Multi Object` - will allow user to fetch data for multiple data extensions, " + + "`Single Object` - will allow user to fetch data for single data extension.") private String queryMode; @Name(PROPERTY_DATA_EXTENSION_KEY_LIST) @Macro @Nullable - @Description("Application name for which data to be fetched. The application can be one of three values: " + - "`Contract Management` - will fetch data for all tables under Contract Management application, " + - "`Product Catalog` - will fetch data for all tables under Product Catalog application, " + - "`Procurement` - will fetch data for all tables under Procurement application. " + - "Note, the Application name value will be ignored if the Mode is set to `Table`.") + @Description("Specify the data extension keys from which data to be fetched; for example: 'Key1,Key2'. " + + "Note, this value will be ignored if the Mode is set to `Single Object`.") private String dataExtensionKeys; @Name(PROPERTY_TABLE_NAME_FIELD) @Macro @Nullable - @Description("The name of the field that holds the table name. Must not be the name of any table column that " + - "will be read. Defaults to `tablename`. Note, the Table name field value will be ignored if the Mode " + - "is set to `Table`.") + @Description("The name of the field that holds the data extension name. Must not be the name of any data " + + "extension column that will be read. Defaults to `tablename`. Note, the Table name field value will be ignored " + + "if the Mode is set to `Single Object`.") private String tableNameField; @Name(PROPERTY_DATA_EXTENSION_KEY) @Macro @Nullable - @Description("The name of the Salesforce table from which data to be fetched. Note, the Table name value " + - "will be ignored if the Mode is set to `Reporting`.") + @Description("Specify the data extension key from which data to be fetched. Note, this value will be ignored if " + + "the Mode is set to `Multi Object`.") private String dataExtensionKey; @Name(PROPERTY_CLIENT_ID) @Macro - @Description(" The Client ID for Salesforce Instance.") + @Description("OAuth2 client ID associated with an installed package in the Salesforce Marketing Cloud.") private String clientId; @Name(PROPERTY_CLIENT_SECRET) @Macro - @Description("The Client Secret for Salesforce Instance.") + @Description("OAuth2 client secret associated with an installed package in the Salesforce Marketing Cloud.") private String clientSecret; @Name(PROPERTY_API_ENDPOINT) @Macro - @Description("The REST API Endpoint for Salesforce Instance. For example, https://instance.service-now.com") + @Description("The REST API Base URL associated for the Server-to-Server API integration. " + + "For example, https://instance.rest.marketingcloudapis.com/") private String restEndpoint; @Name(PROPERTY_AUTH_API_ENDPOINT) @Macro - @Description("The AUTH API Endpoint for Salesforce Instance. " + - "For example, https://instance.auth.marketingcloudapis.com") + @Description("Authentication Base URL associated for the Server-to-Server API integration. " + + "For example, https://instance.auth.marketingcloudapis.com/") private String authEndpoint; @Name(PROPERTY_SOAP_API_ENDPOINT) @Macro - @Description("The SOAP API Endpoint for Salesforce Instance. " + - "For example https://instance.Salesforce.soap.marketingcloudapis.com/Service.asmx") + @Description("The SOAP Endpoint URL associated for the Server-to-Server API integration. " + + "For example, https://instance.soap.marketingcloudapis.com/Service.asmx") private String soapEndpoint; public SalesforceSourceConfig(String referenceName, String queryMode, @Nullable String dataExtensionKeys, @@ -131,20 +130,20 @@ public String getReferenceName() { return referenceName; } - public SourceObjectMode getQueryMode(FailureCollector collector) { - SourceObjectMode mode = getQueryMode(); + public SourceQueryMode getQueryMode(FailureCollector collector) { + SourceQueryMode mode = getQueryMode(); if (mode != null) { return mode; } collector.addFailure("Unsupported query mode value: " + queryMode, - String.format("Supported modes are: %s", SourceObjectMode.getSupportedModes())) + String.format("Supported modes are: %s", SourceQueryMode.getSupportedModes())) .withConfigProperty(PROPERTY_QUERY_MODE); throw collector.getOrThrowException(); } - public SourceObjectMode getQueryMode() { - Optional sourceQueryMode = SourceObjectMode.fromValue(queryMode); + public SourceQueryMode getQueryMode() { + Optional sourceQueryMode = SourceQueryMode.fromValue(queryMode); return sourceQueryMode.isPresent() ? sourceQueryMode.get() : null; } @@ -189,7 +188,7 @@ public String getSoapEndpoint() { */ public void validate(FailureCollector collector) { //Validates the given referenceName to consists of characters allowed to represent a dataset. - //IdUtils.validateReferenceName(referenceName, collector); + IdUtils.validateReferenceName(referenceName, collector); validateCredentials(collector); validateQueryMode(collector); @@ -246,9 +245,9 @@ private void validateQueryMode(FailureCollector collector) { return; } - SourceObjectMode mode = getQueryMode(collector); + SourceQueryMode mode = getQueryMode(collector); - if (mode == SourceObjectMode.MULTI_OBJECT) { + if (mode == SourceQueryMode.MULTI_OBJECT) { validateMultiObjectQueryMode(collector); } else { validateSingleObjectQueryMode(collector); diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SourceObjectMode.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SourceQueryMode.java similarity index 76% rename from src/main/java/io/cdap/plugin/sfmc/source/util/SourceObjectMode.java rename to src/main/java/io/cdap/plugin/sfmc/source/util/SourceQueryMode.java index a6fd5bc..07d8b4e 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SourceObjectMode.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SourceQueryMode.java @@ -24,38 +24,38 @@ /** * Indicates query mode which will be used when fetching Salesforce data. */ -public enum SourceObjectMode { +public enum SourceQueryMode { /** - * Mode will be used as Reporting for querying data + * Mode will be used to fetch data for multiple data extensions */ MULTI_OBJECT("Multi Object"), /** - * Mode will be used as Table for querying data + * Mode will be used to fetch data for single data extension */ SINGLE_OBJECT("Single Object"); private final String value; - SourceObjectMode(String value) { + SourceQueryMode(String value) { this.value = value; } /** - * Converts mode string value into {@link SourceObjectMode} enum. + * Converts mode string value into {@link SourceQueryMode} enum. * * @param stringValue mode string value * @return source query mode in optional container */ - public static Optional fromValue(String stringValue) { + public static Optional fromValue(String stringValue) { return Stream.of(values()) .filter(keyType -> keyType.value.equalsIgnoreCase(stringValue)) .findAny(); } public static String getSupportedModes() { - return Arrays.stream(SourceObjectMode.values()).map(SourceObjectMode::getValue) + return Arrays.stream(SourceQueryMode.values()).map(SourceQueryMode::getValue) .collect(Collectors.joining(", ")); } From 5883e4cac554fd1c61d9c931c9e5ab66f03ae755 Mon Sep 17 00:00:00 2001 From: amolkokane76 Date: Mon, 30 Mar 2020 16:55:04 +0530 Subject: [PATCH 11/19] Added JUnit cases --- docs/SalesforceDataExtension-batchsource.md | 8 + ...alesforce_source_multiobject_pipeline.json | 102 +++++++ ...lesforce_source_singleobject_pipeline.json | 103 +++++++ pom.xml | 12 + .../sfmc/common/DataExtensionClient.java | 2 +- .../{ => sink}/DataExtensionOutputFormat.java | 2 +- .../{ => sink}/DataExtensionRecordWriter.java | 2 +- .../sfmc/{ => sink}/MarketingCloudConf.java | 2 +- .../MarketingCloudDataExtensionSink.java | 2 +- .../plugin/sfmc/{ => sink}/Operation.java | 2 +- .../RecordDataExtensionRowConverter.java | 2 +- .../plugin/sfmc/source/SalesforceSource.java | 8 +- .../sfmc/source/SalesforceSourceConfig.java | 102 +++++-- .../sfmc/source/util/SalesforceConstants.java | 10 + .../plugin/sfmc/source/util/SourceObject.java | 60 ++++ .../source/SalesforceSourceConfigHelper.java | 116 ++++++++ .../source/SalesforceSourceConfigTest.java | 272 ++++++++++++++++++ .../SalesforceDataExtension-batchsource.json | 28 ++ 18 files changed, 807 insertions(+), 28 deletions(-) create mode 100644 examples/example_salesforce_source_multiobject_pipeline.json create mode 100644 examples/example_salesforce_source_singleobject_pipeline.json rename src/main/java/io/cdap/plugin/sfmc/{ => sink}/DataExtensionOutputFormat.java (99%) rename src/main/java/io/cdap/plugin/sfmc/{ => sink}/DataExtensionRecordWriter.java (99%) rename src/main/java/io/cdap/plugin/sfmc/{ => sink}/MarketingCloudConf.java (99%) rename src/main/java/io/cdap/plugin/sfmc/{ => sink}/MarketingCloudDataExtensionSink.java (99%) rename src/main/java/io/cdap/plugin/sfmc/{ => sink}/Operation.java (95%) rename src/main/java/io/cdap/plugin/sfmc/{ => sink}/RecordDataExtensionRowConverter.java (99%) create mode 100644 src/main/java/io/cdap/plugin/sfmc/source/util/SourceObject.java create mode 100644 src/test/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfigHelper.java create mode 100644 src/test/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfigTest.java diff --git a/docs/SalesforceDataExtension-batchsource.md b/docs/SalesforceDataExtension-batchsource.md index b563888..6d23417 100644 --- a/docs/SalesforceDataExtension-batchsource.md +++ b/docs/SalesforceDataExtension-batchsource.md @@ -22,9 +22,17 @@ Properties `Single Object` - will allow user to fetch data for single data extension. +**Object**: Specify the object for which data to be fetched. Note, this value will +be ignored if the Mode is set to `Multi Object`. + **Data Extension External Key**: Specify the data extension key from which data to be fetched. Note, this value will be ignored if the Mode is set to `Multi Object`. +**Object List**: Specify the comma-separated list of objects for which data to be fetched; +for example: 'Object1,Object2'. + +Note, this value will be ignored if the Mode is set to `Single Object`. + **Data Extension External Keys**: Specify the data extension keys from which data to be fetched; for example: 'Key1,Key2'. diff --git a/examples/example_salesforce_source_multiobject_pipeline.json b/examples/example_salesforce_source_multiobject_pipeline.json new file mode 100644 index 0000000..f3f3a1a --- /dev/null +++ b/examples/example_salesforce_source_multiobject_pipeline.json @@ -0,0 +1,102 @@ +{ + "artifact": { + "name": "cdap-data-pipeline", + "version": "6.1.1", + "scope": "SYSTEM", + "label": "Data Pipeline - Batch" + }, + "description": "", + "name": "Test_Salesforce_Source_MultiObject_Mode", + "config": { + "resources": { + "memoryMB": 2048, + "virtualCores": 1 + }, + "driverResources": { + "memoryMB": 2048, + "virtualCores": 1 + }, + "connections": [ + { + "from": "Salesforce Marketing", + "to": "GCS Multi File" + } + ], + "comments": [], + "postActions": [], + "properties": {}, + "processTimingEnabled": true, + "stageLoggingEnabled": false, + "stages": [ + { + "name": "Salesforce Marketing", + "plugin": { + "name": "SalesforceDataExtension", + "type": "batchsource", + "label": "Salesforce Marketing", + "artifact": { + "name": "salesforce-marketing-plugins", + "version": "1.2.0-SNAPSHOT", + "scope": "USER" + }, + "properties": { + "queryMode": "Multi Object", + "tableNameField": "tablename", + "referenceName": "example_sf_sf_source_1", + "clientId": "${CLIENT_ID}", + "clientSecret": "${CLIENT_SECRET}", + "authEndpoint": "${AUTH_ENDPOINT}", + "soapEndpoint": "${SOAP_ENDPOINT}", + "restEndpoint": "${REST_ENDPOINT}", + "dataExtensionKeyList": "${DATA_EXTENSION_KEY_LIST}", + "objectName": "Data Extension", + "objectList": [ + "Data Extension" + ] + } + }, + "outputSchema": [ + { + "name": "etlSchemaBody", + "schema": "" + } + ] + }, + { + "name": "GCS Multi File", + "plugin": { + "name": "GCSMultiFiles", + "type": "batchsink", + "label": "GCS Multi File", + "artifact": { + "name": "google-cloud", + "version": "0.13.2", + "scope": "SYSTEM" + }, + "properties": { + "project": "${GCS_PROJECT_ID}", + "suffix": "yyyy-MM-dd-HH-mm", + "format": "csv", + "serviceFilePath": "${GCS_SERVICE_ACCOUNT_FILE_PATH}", + "compressionCodec": "None", + "splitField": "tablename", + "location": "us", + "referenceName": "example_sf_gcs_multi_sink_1", + "path": "${GCS_BUCKET_PATH}", + "delimiter": "," + } + }, + "inputSchema": [ + { + "name": "Salesforce Marketing", + "schema": "" + } + ] + } + ], + "schedule": "0 * * * *", + "engine": "spark", + "numOfRecordsPreview": 100, + "maxConcurrentRuns": 1 + } +} \ No newline at end of file diff --git a/examples/example_salesforce_source_singleobject_pipeline.json b/examples/example_salesforce_source_singleobject_pipeline.json new file mode 100644 index 0000000..994c4b3 --- /dev/null +++ b/examples/example_salesforce_source_singleobject_pipeline.json @@ -0,0 +1,103 @@ +{ + "artifact": { + "name": "cdap-data-pipeline", + "version": "6.1.1", + "scope": "SYSTEM", + "label": "Data Pipeline - Batch" + }, + "description": "", + "name": "Test_Salesforce_Source_SingleObject_Mode", + "config": { + "resources": { + "memoryMB": 2048, + "virtualCores": 1 + }, + "driverResources": { + "memoryMB": 2048, + "virtualCores": 1 + }, + "connections": [ + { + "from": "Salesforce Marketing", + "to": "File" + } + ], + "comments": [], + "postActions": [], + "properties": {}, + "processTimingEnabled": true, + "stageLoggingEnabled": false, + "stages": [ + { + "name": "Salesforce Marketing", + "plugin": { + "name": "SalesforceDataExtension", + "type": "batchsource", + "label": "Salesforce Marketing", + "artifact": { + "name": "salesforce-marketing-plugins", + "version": "1.2.0-SNAPSHOT", + "scope": "USER" + }, + "properties": { + "queryMode": "Single Object", + "tableNameField": "tablename", + "referenceName": "example_sf_sf_source_2", + "dataExtensionKey": "${DATA_EXTENSION_KEY}", + "clientId": "${CLIENT_ID}", + "clientSecret": "${CLIENT_SECRET}", + "authEndpoint": "${AUTH_ENDPOINT}", + "soapEndpoint": "${SOAP_ENDPOINT}", + "restEndpoint": "${REST_ENDPOINT}", + "objectName": "Data Extension", + "objectList": [ + "Data Extension" + ] + } + }, + "outputSchema": [ + { + "name": "etlSchemaBody", + "schema": "" + } + ] + }, + { + "name": "File", + "plugin": { + "name": "File", + "type": "batchsink", + "label": "File", + "artifact": { + "name": "core-plugins", + "version": "2.3.4", + "scope": "SYSTEM" + }, + "properties": { + "suffix": "${OUTPUT_PATH_SUFFIX}", + "format": "csv", + "referenceName": "example_sf_file_sink_2", + "path": "${OUTPUT_FILE_PATH}", + "delimiter": "," + } + }, + "outputSchema": [ + { + "name": "etlSchemaBody", + "schema": "" + } + ], + "inputSchema": [ + { + "name": "Salesforce Marketing", + "schema": "" + } + ] + } + ], + "schedule": "0 * * * *", + "engine": "spark", + "numOfRecordsPreview": 100, + "maxConcurrentRuns": 1 + } +} \ No newline at end of file diff --git a/pom.xml b/pom.xml index 26e82fd..f3b3d81 100644 --- a/pom.xml +++ b/pom.xml @@ -73,6 +73,12 @@ guava ${guava.version} + + io.cdap.cdap + hydrator-test + ${cdap.version} + provided + org.apache.hadoop hadoop-common @@ -210,6 +216,12 @@ ${junit.version} test + + org.mockito + mockito-core + 2.24.0 + test + diff --git a/src/main/java/io/cdap/plugin/sfmc/common/DataExtensionClient.java b/src/main/java/io/cdap/plugin/sfmc/common/DataExtensionClient.java index dc0268e..38c4ef6 100644 --- a/src/main/java/io/cdap/plugin/sfmc/common/DataExtensionClient.java +++ b/src/main/java/io/cdap/plugin/sfmc/common/DataExtensionClient.java @@ -42,7 +42,7 @@ import com.exacttarget.fuelsdk.internal.UpdateResult; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.etl.api.FailureCollector; -import io.cdap.plugin.sfmc.MarketingCloudConf; +import io.cdap.plugin.sfmc.sink.MarketingCloudConf; import java.util.ArrayList; import java.util.Collection; diff --git a/src/main/java/io/cdap/plugin/sfmc/DataExtensionOutputFormat.java b/src/main/java/io/cdap/plugin/sfmc/sink/DataExtensionOutputFormat.java similarity index 99% rename from src/main/java/io/cdap/plugin/sfmc/DataExtensionOutputFormat.java rename to src/main/java/io/cdap/plugin/sfmc/sink/DataExtensionOutputFormat.java index 750407a..a39df0b 100644 --- a/src/main/java/io/cdap/plugin/sfmc/DataExtensionOutputFormat.java +++ b/src/main/java/io/cdap/plugin/sfmc/sink/DataExtensionOutputFormat.java @@ -14,7 +14,7 @@ * the License. */ -package io.cdap.plugin.sfmc; +package io.cdap.plugin.sfmc.sink; import com.exacttarget.fuelsdk.ETSdkException; import io.cdap.cdap.api.data.format.StructuredRecord; diff --git a/src/main/java/io/cdap/plugin/sfmc/DataExtensionRecordWriter.java b/src/main/java/io/cdap/plugin/sfmc/sink/DataExtensionRecordWriter.java similarity index 99% rename from src/main/java/io/cdap/plugin/sfmc/DataExtensionRecordWriter.java rename to src/main/java/io/cdap/plugin/sfmc/sink/DataExtensionRecordWriter.java index 7f2ffca..7eea8ca 100644 --- a/src/main/java/io/cdap/plugin/sfmc/DataExtensionRecordWriter.java +++ b/src/main/java/io/cdap/plugin/sfmc/sink/DataExtensionRecordWriter.java @@ -14,7 +14,7 @@ * the License. */ -package io.cdap.plugin.sfmc; +package io.cdap.plugin.sfmc.sink; import com.exacttarget.fuelsdk.ETDataExtensionRow; import com.exacttarget.fuelsdk.ETResponse; diff --git a/src/main/java/io/cdap/plugin/sfmc/MarketingCloudConf.java b/src/main/java/io/cdap/plugin/sfmc/sink/MarketingCloudConf.java similarity index 99% rename from src/main/java/io/cdap/plugin/sfmc/MarketingCloudConf.java rename to src/main/java/io/cdap/plugin/sfmc/sink/MarketingCloudConf.java index c8a29b8..0a65d30 100644 --- a/src/main/java/io/cdap/plugin/sfmc/MarketingCloudConf.java +++ b/src/main/java/io/cdap/plugin/sfmc/sink/MarketingCloudConf.java @@ -14,7 +14,7 @@ * the License. */ -package io.cdap.plugin.sfmc; +package io.cdap.plugin.sfmc.sink; import com.exacttarget.fuelsdk.ETSdkException; import io.cdap.cdap.api.annotation.Description; diff --git a/src/main/java/io/cdap/plugin/sfmc/MarketingCloudDataExtensionSink.java b/src/main/java/io/cdap/plugin/sfmc/sink/MarketingCloudDataExtensionSink.java similarity index 99% rename from src/main/java/io/cdap/plugin/sfmc/MarketingCloudDataExtensionSink.java rename to src/main/java/io/cdap/plugin/sfmc/sink/MarketingCloudDataExtensionSink.java index a8936b6..31f0b3d 100644 --- a/src/main/java/io/cdap/plugin/sfmc/MarketingCloudDataExtensionSink.java +++ b/src/main/java/io/cdap/plugin/sfmc/sink/MarketingCloudDataExtensionSink.java @@ -14,7 +14,7 @@ * the License. */ -package io.cdap.plugin.sfmc; +package io.cdap.plugin.sfmc.sink; import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Name; diff --git a/src/main/java/io/cdap/plugin/sfmc/Operation.java b/src/main/java/io/cdap/plugin/sfmc/sink/Operation.java similarity index 95% rename from src/main/java/io/cdap/plugin/sfmc/Operation.java rename to src/main/java/io/cdap/plugin/sfmc/sink/Operation.java index 72ce14e..417e1a5 100644 --- a/src/main/java/io/cdap/plugin/sfmc/Operation.java +++ b/src/main/java/io/cdap/plugin/sfmc/sink/Operation.java @@ -14,7 +14,7 @@ * the License. */ -package io.cdap.plugin.sfmc; +package io.cdap.plugin.sfmc.sink; /** * The type of write operation. diff --git a/src/main/java/io/cdap/plugin/sfmc/RecordDataExtensionRowConverter.java b/src/main/java/io/cdap/plugin/sfmc/sink/RecordDataExtensionRowConverter.java similarity index 99% rename from src/main/java/io/cdap/plugin/sfmc/RecordDataExtensionRowConverter.java rename to src/main/java/io/cdap/plugin/sfmc/sink/RecordDataExtensionRowConverter.java index ef619e2..ae27bea 100644 --- a/src/main/java/io/cdap/plugin/sfmc/RecordDataExtensionRowConverter.java +++ b/src/main/java/io/cdap/plugin/sfmc/sink/RecordDataExtensionRowConverter.java @@ -14,7 +14,7 @@ * the License. */ -package io.cdap.plugin.sfmc; +package io.cdap.plugin.sfmc.sink; import com.exacttarget.fuelsdk.ETDataExtensionColumn; import com.exacttarget.fuelsdk.ETDataExtensionRow; diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java index a7ba356..4edd390 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java @@ -48,13 +48,13 @@ import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.TABLE_PREFIX; /** - * A {@link BatchSource} that reads data from multiple tables in Service Now. + * A {@link BatchSource} that reads data from multiple objects in Salesforce. */ @Plugin(type = BatchSource.PLUGIN_TYPE) @Name(PLUGIN_NAME) -@Description("Reads from multiple tables in Service Now. " + - "Outputs one record for each row in each table, with the table name as a record field. " + - "Also sets a pipeline argument for each table read, which contains the table schema. ") +@Description("Reads from multiple objects in Salesforce. " + + "Outputs one record for each row in each object, with the object name as a record field. " + + "Also sets a pipeline argument for each object read, which contains the object schema.") public class SalesforceSource extends BatchSource { private static final Logger LOG = LoggerFactory.getLogger(SalesforceSource.class); diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java index 25aece5..fbfe258 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java @@ -17,6 +17,7 @@ package io.cdap.plugin.sfmc.source; import com.exacttarget.fuelsdk.ETSdkException; +import com.google.common.annotations.VisibleForTesting; import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Macro; import io.cdap.cdap.api.annotation.Name; @@ -25,6 +26,7 @@ import io.cdap.plugin.common.Constants; import io.cdap.plugin.common.IdUtils; import io.cdap.plugin.sfmc.common.DataExtensionClient; +import io.cdap.plugin.sfmc.source.util.SourceObject; import io.cdap.plugin.sfmc.source.util.SourceQueryMode; import io.cdap.plugin.sfmc.source.util.Util; import org.slf4j.Logger; @@ -40,6 +42,8 @@ import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_CLIENT_SECRET; import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_DATA_EXTENSION_KEY; import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_DATA_EXTENSION_KEY_LIST; +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_OBJECT_LIST; +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_OBJECT_NAME; import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_QUERY_MODE; import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_SOAP_API_ENDPOINT; import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_TABLE_NAME_FIELD; @@ -61,6 +65,27 @@ public class SalesforceSourceConfig extends PluginConfig { + "`Single Object` - will allow user to fetch data for single data extension.") private String queryMode; + @Name(PROPERTY_OBJECT_NAME) + @Macro + @Nullable + @Description("Specify the object for which data to be fetched. Note, this value will be ignored if the Mode is " + + "set to `Multi Object`.") + private String objectName; + + @Name(PROPERTY_DATA_EXTENSION_KEY) + @Macro + @Nullable + @Description("Specify the data extension key from which data to be fetched. Note, this value will be ignored if " + + "the Mode is set to `Multi Object`.") + private String dataExtensionKey; + + @Name(PROPERTY_OBJECT_LIST) + @Macro + @Nullable + @Description("Specify the comma-separated list of objects for which data to be fetched. Note, this value will be " + + "ignored if the Mode is set to `Single Object`.") + private String objectList; + @Name(PROPERTY_DATA_EXTENSION_KEY_LIST) @Macro @Nullable @@ -76,13 +101,6 @@ public class SalesforceSourceConfig extends PluginConfig { "if the Mode is set to `Single Object`.") private String tableNameField; - @Name(PROPERTY_DATA_EXTENSION_KEY) - @Macro - @Nullable - @Description("Specify the data extension key from which data to be fetched. Note, this value will be ignored if " + - "the Mode is set to `Multi Object`.") - private String dataExtensionKey; - @Name(PROPERTY_CLIENT_ID) @Macro @Description("OAuth2 client ID associated with an installed package in the Salesforce Marketing Cloud.") @@ -111,14 +129,17 @@ public class SalesforceSourceConfig extends PluginConfig { "For example, https://instance.soap.marketingcloudapis.com/Service.asmx") private String soapEndpoint; - public SalesforceSourceConfig(String referenceName, String queryMode, @Nullable String dataExtensionKeys, - @Nullable String tableNameField, @Nullable String dataExtensionKey, String clientId, + public SalesforceSourceConfig(String referenceName, String queryMode, @Nullable String objectName, + @Nullable String dataExtensionKey, @Nullable String objectList, + @Nullable String dataExtensionKeys, @Nullable String tableNameField, String clientId, String clientSecret, String restEndpoint, String authEndpoint, String soapEndpoint) { this.referenceName = referenceName; this.queryMode = queryMode; + this.objectName = objectName; + this.dataExtensionKey = dataExtensionKey; + this.objectList = objectList; this.dataExtensionKeys = dataExtensionKeys; this.tableNameField = tableNameField; - this.dataExtensionKey = dataExtensionKey; this.clientId = clientId; this.clientSecret = clientSecret; this.restEndpoint = restEndpoint; @@ -148,6 +169,20 @@ public SourceQueryMode getQueryMode() { return sourceQueryMode.isPresent() ? sourceQueryMode.get() : null; } + @Nullable + public String getObjectName() { + return objectName; + } + + @Nullable + public String getDataExtensionKey() { + return dataExtensionKey; + } + + public String getObjectList() { + return objectList; + } + @Nullable public String getDataExtensionKeys() { return dataExtensionKeys; @@ -158,11 +193,6 @@ public String getTableNameField() { return tableNameField; } - @Nullable - public String getDataExtensionKey() { - return dataExtensionKey; - } - public String getClientId() { return clientId; } @@ -224,6 +254,12 @@ private void validateCredentials(FailureCollector collector) { .withConfigProperty(PROPERTY_SOAP_API_ENDPOINT); } + collector.getOrThrowException(); + validateSalesforceConnection(collector); + } + + @VisibleForTesting + void validateSalesforceConnection(FailureCollector collector) { try { DataExtensionClient.create("", clientId, clientSecret, authEndpoint, soapEndpoint); } catch (ETSdkException e) { @@ -255,10 +291,28 @@ private void validateQueryMode(FailureCollector collector) { } private void validateMultiObjectQueryMode(FailureCollector collector) { - if (containsMacro(PROPERTY_DATA_EXTENSION_KEY_LIST) || containsMacro(PROPERTY_TABLE_NAME_FIELD)) { + if (containsMacro(PROPERTY_OBJECT_LIST) || containsMacro(PROPERTY_DATA_EXTENSION_KEY_LIST) + || containsMacro(PROPERTY_TABLE_NAME_FIELD)) { return; } + List objects = Util.splitToList(getObjectList(), ','); + if (objects.isEmpty()) { + collector.addFailure("At least 1 Object must be specified.", null) + .withConfigProperty(PROPERTY_OBJECT_LIST); + } + + for (String o : objects) { + Optional sourceObject = SourceObject.fromValue(o); + SourceObject object = sourceObject.isPresent() ? sourceObject.get() : null; + if (object == null) { + collector.addFailure("Unsupported object value: " + o, + String.format("Supported objects are: %s", SourceObject.getSupportedObjects())) + .withConfigProperty(PROPERTY_OBJECT_LIST); + throw collector.getOrThrowException(); + } + } + List dataExtensionKeyList = Util.splitToList(getDataExtensionKeys(), ','); if (dataExtensionKeyList.isEmpty()) { collector.addFailure("At least 1 Data Extension Key must be specified.", null) @@ -272,10 +326,24 @@ private void validateMultiObjectQueryMode(FailureCollector collector) { } private void validateSingleObjectQueryMode(FailureCollector collector) { - if (containsMacro(PROPERTY_DATA_EXTENSION_KEY)) { + if (containsMacro(PROPERTY_OBJECT_NAME) || containsMacro(PROPERTY_DATA_EXTENSION_KEY)) { return; } + if (Util.isNullOrEmpty(objectName)) { + collector.addFailure("Object Name must be specified.", null) + .withConfigProperty(PROPERTY_OBJECT_NAME); + } + + Optional sourceObject = SourceObject.fromValue(objectName); + SourceObject object = sourceObject.isPresent() ? sourceObject.get() : null; + if (object == null) { + collector.addFailure("Unsupported object value: " + objectName, + String.format("Supported objects are: %s", SourceObject.getSupportedObjects())) + .withConfigProperty(PROPERTY_OBJECT_NAME); + throw collector.getOrThrowException(); + } + if (Util.isNullOrEmpty(dataExtensionKey)) { collector.addFailure("Data Extension Key must be specified.", null) .withConfigProperty(PROPERTY_DATA_EXTENSION_KEY); diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java index d8c25d5..eea2397 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java @@ -31,11 +31,21 @@ public interface SalesforceConstants { */ String PROPERTY_QUERY_MODE = "queryMode"; + /** + * Configuration property name used to specify the object name + */ + String PROPERTY_OBJECT_NAME = "objectName"; + /** * Configuration property name used to specify data extension key */ String PROPERTY_DATA_EXTENSION_KEY = "dataExtensionKey"; + /** + * Configuration property name used to specify the list of objects + */ + String PROPERTY_OBJECT_LIST = "objectList"; + /** * Configuration property name used to specify the comma-separated list of data extension keys. */ diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SourceObject.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SourceObject.java new file mode 100644 index 0000000..d71d394 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SourceObject.java @@ -0,0 +1,60 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.sfmc.source.util; + +import java.util.Arrays; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Indicates the object for which data to be fetched from Salesforce. + */ +public enum SourceObject { + + /** + * This indicates data to be fetched from Data Extension + */ + DATA_EXTENSION("Data Extension"); + + private final String value; + + SourceObject(String value) { + this.value = value; + } + + /** + * Converts object string value into {@link SourceObject} enum. + * + * @param stringValue object string value + * @return source object in optional container + */ + public static Optional fromValue(String stringValue) { + return Stream.of(values()) + .filter(keyType -> keyType.value.equalsIgnoreCase(stringValue)) + .findAny(); + } + + public static String getSupportedObjects() { + return Arrays.stream(SourceObject.values()).map(SourceObject::getValue) + .collect(Collectors.joining(", ")); + } + + public String getValue() { + return value; + } +} diff --git a/src/test/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfigHelper.java b/src/test/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfigHelper.java new file mode 100644 index 0000000..0ca4aff --- /dev/null +++ b/src/test/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfigHelper.java @@ -0,0 +1,116 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.sfmc.source; + +/** + * Utility class that provides handy methods to construct Salesforce Source Config for testing + */ +public class SalesforceSourceConfigHelper { + + public static final String TEST_REF_NAME = "TestRefName"; + public static final String TEST_CLIENT_ID = "test-client-id"; + public static final String TEST_CLIENT_SECRET = "test-client-secret"; + public static final String TEST_REST_ENDPOINT = "TestRestEndpoint"; + public static final String TEST_AUTH_ENDPOINT = "TestAuthEndpoint"; + public static final String TEST_SOAP_ENDPOINT = "TestSoapEndpoint"; + public static final String TEST_OBJECT_NAME = "Data Extension"; + public static final String TEST_OBJECT_LIST = "Data Extension"; + + public static ConfigBuilder newConfigBuilder() { + return new ConfigBuilder(); + } + + public static class ConfigBuilder { + private String referenceName = TEST_REF_NAME; + private String clientId = TEST_CLIENT_ID; + private String clientSecret = TEST_CLIENT_SECRET; + private String restEndpoint = TEST_REST_ENDPOINT; + private String authEndpoint = TEST_AUTH_ENDPOINT; + private String soapEndpoint = TEST_SOAP_ENDPOINT; + private String queryMode; + private String objectName = TEST_OBJECT_NAME; + private String dataExtensionKey; + private String objectList = TEST_OBJECT_LIST; + private String dataExtensionKeys; + private String tableNameField = "tablename"; + + public ConfigBuilder setReferenceName(String referenceName) { + this.referenceName = referenceName; + return this; + } + + public ConfigBuilder setQueryMode(String queryMode) { + this.queryMode = queryMode; + return this; + } + + public ConfigBuilder setObjectName(String objectName) { + this.objectName = objectName; + return this; + } + + public ConfigBuilder setDataExtensionKey(String dataExtensionKey) { + this.dataExtensionKey = dataExtensionKey; + return this; + } + + public ConfigBuilder setObjectList(String objectList) { + this.objectList = objectList; + return this; + } + + public ConfigBuilder setDataExtensionKeys(String dataExtensionKeys) { + this.dataExtensionKeys = dataExtensionKeys; + return this; + } + + public ConfigBuilder setTableNameField(String tableNameField) { + this.tableNameField = tableNameField; + return this; + } + + public ConfigBuilder setClientId(String clientId) { + this.clientId = clientId; + return this; + } + + public ConfigBuilder setClientSecret(String clientSecret) { + this.clientSecret = clientSecret; + return this; + } + + public ConfigBuilder setRestEndpoint(String restEndpoint) { + this.restEndpoint = restEndpoint; + return this; + } + + public ConfigBuilder setAuthEndpoint(String authEndpoint) { + this.authEndpoint = authEndpoint; + return this; + } + + public ConfigBuilder setSoapEndpoint(String soapEndpoint) { + this.soapEndpoint = soapEndpoint; + return this; + } + + public SalesforceSourceConfig build() { + return new SalesforceSourceConfig(referenceName, queryMode, objectName, dataExtensionKey, objectList, + dataExtensionKeys, tableNameField, clientId, clientSecret, restEndpoint, authEndpoint, soapEndpoint); + } + } +} diff --git a/src/test/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfigTest.java b/src/test/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfigTest.java new file mode 100644 index 0000000..0bffbd5 --- /dev/null +++ b/src/test/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfigTest.java @@ -0,0 +1,272 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.sfmc.source; + +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.cdap.etl.api.validation.CauseAttributes; +import io.cdap.cdap.etl.api.validation.ValidationException; +import io.cdap.cdap.etl.mock.validation.MockFailureCollector; +import io.cdap.plugin.sfmc.source.util.SourceQueryMode; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.mockito.Mockito; + +import static io.cdap.plugin.sfmc.source.SalesforceSourceConfigHelper.TEST_AUTH_ENDPOINT; +import static io.cdap.plugin.sfmc.source.SalesforceSourceConfigHelper.TEST_CLIENT_ID; +import static io.cdap.plugin.sfmc.source.SalesforceSourceConfigHelper.TEST_CLIENT_SECRET; +import static io.cdap.plugin.sfmc.source.SalesforceSourceConfigHelper.TEST_REST_ENDPOINT; +import static io.cdap.plugin.sfmc.source.SalesforceSourceConfigHelper.TEST_SOAP_ENDPOINT; +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_AUTH_API_ENDPOINT; +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_CLIENT_ID; +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_CLIENT_SECRET; +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_DATA_EXTENSION_KEY; +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_DATA_EXTENSION_KEY_LIST; +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_QUERY_MODE; +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_SOAP_API_ENDPOINT; +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_TABLE_NAME_FIELD; + +/** + * Tests for {@link SalesforceSourceConfig}. + */ +public class SalesforceSourceConfigTest { + + @Rule + public ExpectedException thrown = ExpectedException.none(); + + @Test + public void testQueryModeSingleObject() { + SourceQueryMode queryMode = SourceQueryMode.SINGLE_OBJECT; + SalesforceSourceConfig config = SalesforceSourceConfigHelper.newConfigBuilder() + .setQueryMode("Single Object") + .build(); + + MockFailureCollector collector = new MockFailureCollector(); + Assert.assertEquals(queryMode, config.getQueryMode(collector)); + Assert.assertEquals(0, collector.getValidationFailures().size()); + } + + @Test + public void testQueryModeMultiObject() { + SourceQueryMode queryMode = SourceQueryMode.MULTI_OBJECT; + SalesforceSourceConfig config = SalesforceSourceConfigHelper.newConfigBuilder() + .setQueryMode("Multi Object") + .build(); + + MockFailureCollector collector = new MockFailureCollector(); + Assert.assertEquals(queryMode, config.getQueryMode(collector)); + Assert.assertEquals(0, collector.getValidationFailures().size()); + } + + @Test + public void testQueryModeInvalid() { + SalesforceSourceConfig config = SalesforceSourceConfigHelper.newConfigBuilder() + .setQueryMode(null) + .build(); + + try { + MockFailureCollector collector = new MockFailureCollector(); + config.getQueryMode(collector); + } catch (ValidationException e) { + Assert.assertEquals(1, e.getFailures().size()); + Assert.assertEquals(PROPERTY_QUERY_MODE, e.getFailures().get(0).getCauses().get(0) + .getAttribute(CauseAttributes.STAGE_CONFIG)); + } + } + + @Test + public void testValidateClientIdNull() { + MockFailureCollector collector = new MockFailureCollector(); + SalesforceSourceConfig config = withSalesforceValidationMock(SalesforceSourceConfigHelper.newConfigBuilder() + .setQueryMode("Single Object") + .setClientId(null) + .setClientSecret(TEST_CLIENT_SECRET) + .setRestEndpoint(TEST_REST_ENDPOINT) + .setAuthEndpoint(TEST_AUTH_ENDPOINT) + .setSoapEndpoint(TEST_SOAP_ENDPOINT) + .build(), collector); + + try { + config.validate(collector); + collector.getOrThrowException(); + } catch (ValidationException e) { + Assert.assertEquals(1, e.getFailures().size()); + Assert.assertEquals(PROPERTY_CLIENT_ID, e.getFailures().get(0).getCauses().get(0) + .getAttribute(CauseAttributes.STAGE_CONFIG)); + } + } + + @Test + public void testValidateClientSecretNull() { + MockFailureCollector collector = new MockFailureCollector(); + SalesforceSourceConfig config = withSalesforceValidationMock(SalesforceSourceConfigHelper.newConfigBuilder() + .setQueryMode("Single Object") + .setClientId(TEST_CLIENT_ID) + .setClientSecret(null) + .setRestEndpoint(TEST_REST_ENDPOINT) + .setAuthEndpoint(TEST_AUTH_ENDPOINT) + .setSoapEndpoint(TEST_SOAP_ENDPOINT) + .build(), collector); + + try { + config.validate(collector); + collector.getOrThrowException(); + } catch (ValidationException e) { + Assert.assertEquals(1, e.getFailures().size()); + Assert.assertEquals(PROPERTY_CLIENT_SECRET, e.getFailures().get(0).getCauses().get(0) + .getAttribute(CauseAttributes.STAGE_CONFIG)); + } + } + + @Test + public void testValidateAuthEndpointNull() { + MockFailureCollector collector = new MockFailureCollector(); + SalesforceSourceConfig config = withSalesforceValidationMock(SalesforceSourceConfigHelper.newConfigBuilder() + .setQueryMode("Single Object") + .setClientId(TEST_CLIENT_ID) + .setClientSecret(TEST_CLIENT_SECRET) + .setRestEndpoint(TEST_REST_ENDPOINT) + .setAuthEndpoint(null) + .setSoapEndpoint(TEST_SOAP_ENDPOINT) + .build(), collector); + + try { + config.validate(collector); + collector.getOrThrowException(); + } catch (ValidationException e) { + Assert.assertEquals(1, e.getFailures().size()); + Assert.assertEquals(PROPERTY_AUTH_API_ENDPOINT, e.getFailures().get(0).getCauses().get(0) + .getAttribute(CauseAttributes.STAGE_CONFIG)); + } + } + + @Test + public void testValidateSoapEndpointNull() { + MockFailureCollector collector = new MockFailureCollector(); + SalesforceSourceConfig config = withSalesforceValidationMock(SalesforceSourceConfigHelper.newConfigBuilder() + .setQueryMode("Single Object") + .setClientId(TEST_CLIENT_ID) + .setClientSecret(TEST_CLIENT_SECRET) + .setRestEndpoint(TEST_REST_ENDPOINT) + .setAuthEndpoint(TEST_AUTH_ENDPOINT) + .setSoapEndpoint(null) + .build(), collector); + + try { + config.validate(collector); + collector.getOrThrowException(); + } catch (ValidationException e) { + Assert.assertEquals(1, e.getFailures().size()); + Assert.assertEquals(PROPERTY_SOAP_API_ENDPOINT, e.getFailures().get(0).getCauses().get(0) + .getAttribute(CauseAttributes.STAGE_CONFIG)); + } + } + + @Test + public void testValidCredentials() { + MockFailureCollector collector = new MockFailureCollector(); + SalesforceSourceConfig config = withSalesforceValidationMock(SalesforceSourceConfigHelper.newConfigBuilder() + .setQueryMode("Single Object") + .setClientId(TEST_CLIENT_ID) + .setClientSecret(TEST_CLIENT_SECRET) + .setRestEndpoint(TEST_REST_ENDPOINT) + .setAuthEndpoint(TEST_AUTH_ENDPOINT) + .setSoapEndpoint(TEST_SOAP_ENDPOINT) + .setDataExtensionKey("test-dataextension-key") + .build(), collector); + + config.validate(collector); + + Assert.assertEquals(0, collector.getValidationFailures().size()); + } + + @Test + public void testSingleObjectModeMissingDataExtensionKey() { + MockFailureCollector collector = new MockFailureCollector(); + SalesforceSourceConfig config = withSalesforceValidationMock(SalesforceSourceConfigHelper.newConfigBuilder() + .setQueryMode("Single Object") + .setRestEndpoint(TEST_REST_ENDPOINT) + .setAuthEndpoint(TEST_AUTH_ENDPOINT) + .setSoapEndpoint(TEST_SOAP_ENDPOINT) + .setDataExtensionKey(null) + .build(), collector); + + try { + config.validate(collector); + collector.getOrThrowException(); + } catch (ValidationException e) { + Assert.assertEquals(1, e.getFailures().size()); + Assert.assertEquals(PROPERTY_DATA_EXTENSION_KEY, e.getFailures().get(0).getCauses().get(0) + .getAttribute(CauseAttributes.STAGE_CONFIG)); + } + } + + @Test + public void testMultiObjectModeMissingDataExtensionKeys() { + MockFailureCollector collector = new MockFailureCollector(); + SalesforceSourceConfig config = withSalesforceValidationMock(SalesforceSourceConfigHelper.newConfigBuilder() + .setQueryMode("Multi Object") + .setClientId(TEST_CLIENT_ID) + .setClientSecret(TEST_CLIENT_SECRET) + .setRestEndpoint(TEST_REST_ENDPOINT) + .setAuthEndpoint(TEST_AUTH_ENDPOINT) + .setSoapEndpoint(TEST_SOAP_ENDPOINT) + .setDataExtensionKeys(null) + .build(), collector); + + try { + config.validate(collector); + collector.getOrThrowException(); + } catch (ValidationException e) { + Assert.assertEquals(1, e.getFailures().size()); + Assert.assertEquals(PROPERTY_DATA_EXTENSION_KEY_LIST, e.getFailures().get(0).getCauses().get(0) + .getAttribute(CauseAttributes.STAGE_CONFIG)); + } + } + + @Test + public void testMultiObjectModeMissingTableNameField() { + MockFailureCollector collector = new MockFailureCollector(); + SalesforceSourceConfig config = withSalesforceValidationMock(SalesforceSourceConfigHelper.newConfigBuilder() + .setQueryMode("Multi Object") + .setClientId(TEST_CLIENT_ID) + .setClientSecret(TEST_CLIENT_SECRET) + .setRestEndpoint(TEST_REST_ENDPOINT) + .setAuthEndpoint(TEST_AUTH_ENDPOINT) + .setSoapEndpoint(TEST_SOAP_ENDPOINT) + .setDataExtensionKeys("Test-DataExtension-Key1") + .setTableNameField(null) + .build(), collector); + + try { + config.validate(collector); + collector.getOrThrowException(); + } catch (ValidationException e) { + Assert.assertEquals(1, e.getFailures().size()); + Assert.assertEquals(PROPERTY_TABLE_NAME_FIELD, e.getFailures().get(0).getCauses().get(0) + .getAttribute(CauseAttributes.STAGE_CONFIG)); + } + } + + private SalesforceSourceConfig withSalesforceValidationMock(SalesforceSourceConfig config, + FailureCollector collector) { + SalesforceSourceConfig spy = Mockito.spy(config); + Mockito.doNothing().when(spy).validateSalesforceConnection(collector); + return spy; + } +} diff --git a/widgets/SalesforceDataExtension-batchsource.json b/widgets/SalesforceDataExtension-batchsource.json index 1753f33..dee9fcb 100644 --- a/widgets/SalesforceDataExtension-batchsource.json +++ b/widgets/SalesforceDataExtension-batchsource.json @@ -32,6 +32,17 @@ { "label": "Single Object Retrieval", "properties": [ + { + "widget-type": "select", + "label": "Object", + "name": "objectName", + "widget-attributes": { + "values": [ + "Data Extension" + ], + "default": "Data Extension" + } + }, { "widget-type": "textbox", "label": "Data Extension External Key", @@ -45,6 +56,23 @@ { "label": "Multi Object Retrieval", "properties": [ + { + "widget-type": "multi-select", + "label": "Object List", + "name": "objectList", + "widget-attributes": { + "options": [ + { + "id": "Data Extension", + "label": "Data Extension" + } + ], + "delimiter": ",", + "default": [ + "Data Extension" + ] + } + }, { "widget-type": "csv", "label": "Data Extension External Keys", From 988ba2a3bf7698751b1cbdf823c7ff6066d8eddf Mon Sep 17 00:00:00 2001 From: amolkokane76 Date: Mon, 6 Apr 2020 09:51:31 +0530 Subject: [PATCH 12/19] Added code to support data extraction from multiple objects --- README.md | 2 +- docs/Salesforce-batchsource.md | 78 +++++++++ docs/SalesforceDataExtension-batchsource.md | 56 ------ ...alesforce_source_multiobject_pipeline.json | 6 +- ...lesforce_source_singleobject_pipeline.json | 8 +- ...hsource.png => Salesforce-batchsource.png} | Bin pom.xml | 2 +- .../{common => sink}/DataExtensionClient.java | 70 ++------ .../{common => sink}/DataExtensionInfo.java | 2 +- .../sfmc/sink/DataExtensionOutputFormat.java | 1 - .../sfmc/sink/DataExtensionRecordWriter.java | 1 - .../plugin/sfmc/sink/MarketingCloudConf.java | 1 - .../sink/RecordDataExtensionRowConverter.java | 3 +- .../plugin/sfmc/source/SalesforceClient.java | 165 ++++++++++++++++++ .../sfmc/source/SalesforceInputFormat.java | 128 ++++++-------- .../sfmc/source/SalesforceInputSplit.java | 14 +- .../sfmc/source/SalesforceRecordReader.java | 118 +++++++++---- .../plugin/sfmc/source/SalesforceSource.java | 13 +- .../sfmc/source/SalesforceSourceConfig.java | 139 ++++++++++----- .../sfmc/source/util/SalesforceColumn.java | 5 - .../sfmc/source/util/SalesforceConstants.java | 4 +- .../source/util/SalesforceObjectInfo.java | 32 +++- .../sfmc/source/util/SchemaBuilder.java | 2 + .../plugin/sfmc/source/util/SourceObject.java | 37 +++- .../sfmc/source/util/SourceQueryMode.java | 4 +- ...ource.json => Salesforce-batchsource.json} | 21 ++- 26 files changed, 588 insertions(+), 324 deletions(-) create mode 100644 docs/Salesforce-batchsource.md delete mode 100644 docs/SalesforceDataExtension-batchsource.md rename icons/{SalesforceDataExtension-batchsource.png => Salesforce-batchsource.png} (100%) rename src/main/java/io/cdap/plugin/sfmc/{common => sink}/DataExtensionClient.java (89%) rename src/main/java/io/cdap/plugin/sfmc/{common => sink}/DataExtensionInfo.java (97%) create mode 100644 src/main/java/io/cdap/plugin/sfmc/source/SalesforceClient.java rename widgets/{SalesforceDataExtension-batchsource.json => Salesforce-batchsource.json} (89%) diff --git a/README.md b/README.md index 75d37cf..573c64d 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Contains CDAP pipeline plugins for Salesforce Marketing Cloud. Following plugins are available in this repository. -* [Salesforce Marketing Cloud Data Extension Source](docs/SalesforceDataExtension-batchsource.md) +* [Salesforce Marketing Cloud Source](docs/Salesforce-batchsource.md) * [Salesforce Marketing Cloud Data Extension Sink](docs/SalesforceDataExtension-batchsink.md) # Contact diff --git a/docs/Salesforce-batchsource.md b/docs/Salesforce-batchsource.md new file mode 100644 index 0000000..bc32bef --- /dev/null +++ b/docs/Salesforce-batchsource.md @@ -0,0 +1,78 @@ +# Salesforce Marketing Cloud Source + +Description +----------- + +Reads from one or multiple objects withing Salesforce Marketing Cloud depending on the mode values set for this plugin. +In case of `Multi Object` mode, the source will output a record for each row from the object it reads, with each record +containing an additional field that holds the name of the object from which the record came from. In case of +`Single Object` mode, this additional field will not be there in the output. In addition, for each object that will be +read, this plugin will set pipeline arguments where the key is `multisink.[objectname]` and the value is the schema of +the object. + +Properties +---------- + +**Reference Name**: Name used to uniquely identify this source for lineage, annotating metadata, etc. + +**Data Retrieval Mode**: Mode of data retrieval. The mode can be one of two values: + +`Multi Object` - will allow user to fetch data for multiple data extensions, + +`Single Object` - will allow user to fetch data for single data extension. + +**Object**: Specify the object for which data to be fetched. This can be one of following values: + +`Data Extension` - will allow user to fetch data for a single Data Extension object, + +`Campaign` - will allow user to fetch data for Campaign object, + +`Email` - will allow user to fetch data for Email object, + +`Mailing List` - will allow user to fetch data for Mailing List object. + +Note, this value will be ignored if the Mode is set to `Multi Object`. + +**Data Extension External Key**: Specify the data extension key from which data to be fetched. Note, this value will +be ignored in following two cases: + +* If the Mode is set to `Multi Object` +* If the selected object name is other than `Data Extension`. + +**Object List**: Specify the comma-separated list of objects for which data to be fetched; for example: +'Object1,Object2'. This can be one or more values from following possible values: + +`Data Extension` - will allow user to fetch data for a single Data Extension object, + +`Campaign` - will allow user to fetch data for Campaign object, + +`Email` - will allow user to fetch data for Email object, + +`Mailing List` - will allow user to fetch data for Mailing List object. + +Note, this value will be ignored if the Mode is set to `Single Object`. + +**Data Extension External Keys**: Specify the data extension keys from which data to be fetched; for example: +'Key1,Key2'. Note, this value will be ignored in following two cases: + +* If the Mode is set to `Single Object`, + +* If the selected object list does not contain `Data Extension` as one of the objects. + +**Table Name Field**: The name of the field that holds the object name to which the data belongs to. Must not be the +name of any column for any of the objects that will be read. Defaults to `tablename`. In case of `Data Extension` +object, this field will have value in `dataextension-[Data Extension Key]` format. Note, the Table name field value +will be ignored if the Mode is set to `Single Object`. + +**Client ID**: OAuth2 client ID associated with an installed package in the Salesforce Marketing Cloud. + +**Client Secret**: TOAuth2 client secret associated with an installed package in the Salesforce Marketing Cloud. + +**Authentication Base URI**: Authentication Base URL associated for the Server-to-Server API integration. +For example, `https://instance.auth.marketingcloudapis.com/` + +**SOAP API Endpoint**: The SOAP Endpoint URL associated for the Server-to-Server API integration. For example, +`https://instance.soap.marketingcloudapis.com/Service.asmx` + +**REST API Base URI**: The REST API Base URL associated for the Server-to-Server API integration. For example, +`https://instance.rest.marketingcloudapis.com/` diff --git a/docs/SalesforceDataExtension-batchsource.md b/docs/SalesforceDataExtension-batchsource.md deleted file mode 100644 index 6d23417..0000000 --- a/docs/SalesforceDataExtension-batchsource.md +++ /dev/null @@ -1,56 +0,0 @@ -# Salesforce Marketing Cloud Data Extension Source - -Description ------------ - -Reads from one or multiple data extensions within Salesforce Marketing Cloud depending on the mode values set for -this plugin. In case of `Multi Object` mode, the source will output a record for each row in the data extension it -reads, with each record containing an additional field that holds the name of the data extension from which the record -came from. In case of `Single Object` mode, this additional field will not be there in the output. In addition, for -each data extension that will be read, this plugin will set pipeline arguments where the key is -`multisink.[dataextensionname]` and the value is the schema of the data extension. - - -Properties ----------- - -**Reference Name**: Name used to uniquely identify this source for lineage, annotating metadata, etc. - -**Data Retrieval Mode**: Mode of data retrieval. The mode can be one of two values: - -`Multi Object` - will allow user to fetch data for multiple data extensions, - -`Single Object` - will allow user to fetch data for single data extension. - -**Object**: Specify the object for which data to be fetched. Note, this value will -be ignored if the Mode is set to `Multi Object`. - -**Data Extension External Key**: Specify the data extension key from which data to be fetched. Note, this value will -be ignored if the Mode is set to `Multi Object`. - -**Object List**: Specify the comma-separated list of objects for which data to be fetched; -for example: 'Object1,Object2'. - -Note, this value will be ignored if the Mode is set to `Single Object`. - -**Data Extension External Keys**: Specify the data extension keys from which data to be fetched; -for example: 'Key1,Key2'. - -Note, this value will be ignored if the Mode is set to `Single Object`. - -**Table Name Field**: The name of the field that holds the data extension name. Must not be the name of any data -extension column that will be read. Defaults to `tablename`. Note, the Table name field value will be ignored if the -Mode is set to `Single Object`. - -**Client ID**: OAuth2 client ID associated with an installed package in the Salesforce Marketing Cloud. - -**Client Secret**: TOAuth2 client secret associated with an installed package in the Salesforce Marketing Cloud. - -**Authentication Base URI**: Authentication Base URL associated for the Server-to-Server API integration. -For example, `https://instance.auth.marketingcloudapis.com/` - -**SOAP API Endpoint**: The SOAP Endpoint URL associated for the Server-to-Server API integration. For example, -`https://instance.soap.marketingcloudapis.com/Service.asmx` - -**REST API Base URI**: The REST API Base URL associated for the Server-to-Server API integration. For example, -`https://instance.rest.marketingcloudapis.com/` diff --git a/examples/example_salesforce_source_multiobject_pipeline.json b/examples/example_salesforce_source_multiobject_pipeline.json index f3f3a1a..b539f29 100644 --- a/examples/example_salesforce_source_multiobject_pipeline.json +++ b/examples/example_salesforce_source_multiobject_pipeline.json @@ -31,7 +31,7 @@ { "name": "Salesforce Marketing", "plugin": { - "name": "SalesforceDataExtension", + "name": "Salesforce", "type": "batchsource", "label": "Salesforce Marketing", "artifact": { @@ -50,9 +50,7 @@ "restEndpoint": "${REST_ENDPOINT}", "dataExtensionKeyList": "${DATA_EXTENSION_KEY_LIST}", "objectName": "Data Extension", - "objectList": [ - "Data Extension" - ] + "objectList": "${SFMC_OBJECT_LIST}" } }, "outputSchema": [ diff --git a/examples/example_salesforce_source_singleobject_pipeline.json b/examples/example_salesforce_source_singleobject_pipeline.json index 994c4b3..2c4fd39 100644 --- a/examples/example_salesforce_source_singleobject_pipeline.json +++ b/examples/example_salesforce_source_singleobject_pipeline.json @@ -31,7 +31,7 @@ { "name": "Salesforce Marketing", "plugin": { - "name": "SalesforceDataExtension", + "name": "Salesforce", "type": "batchsource", "label": "Salesforce Marketing", "artifact": { @@ -49,10 +49,8 @@ "authEndpoint": "${AUTH_ENDPOINT}", "soapEndpoint": "${SOAP_ENDPOINT}", "restEndpoint": "${REST_ENDPOINT}", - "objectName": "Data Extension", - "objectList": [ - "Data Extension" - ] + "objectName": "${SFMC_OBJECT_NAME}", + "objectList": "Data Extension" } }, "outputSchema": [ diff --git a/icons/SalesforceDataExtension-batchsource.png b/icons/Salesforce-batchsource.png similarity index 100% rename from icons/SalesforceDataExtension-batchsource.png rename to icons/Salesforce-batchsource.png diff --git a/pom.xml b/pom.xml index f3b3d81..c8334ab 100644 --- a/pom.xml +++ b/pom.xml @@ -36,7 +36,7 @@ 2.3.0 2.2.0 27.0.1-jre - 1.4.0 + 1.5.0 diff --git a/src/main/java/io/cdap/plugin/sfmc/common/DataExtensionClient.java b/src/main/java/io/cdap/plugin/sfmc/sink/DataExtensionClient.java similarity index 89% rename from src/main/java/io/cdap/plugin/sfmc/common/DataExtensionClient.java rename to src/main/java/io/cdap/plugin/sfmc/sink/DataExtensionClient.java index 38c4ef6..303c08b 100644 --- a/src/main/java/io/cdap/plugin/sfmc/common/DataExtensionClient.java +++ b/src/main/java/io/cdap/plugin/sfmc/sink/DataExtensionClient.java @@ -14,15 +14,13 @@ * the License. */ -package io.cdap.plugin.sfmc.common; +package io.cdap.plugin.sfmc.sink; import com.exacttarget.fuelsdk.ETClient; import com.exacttarget.fuelsdk.ETConfiguration; import com.exacttarget.fuelsdk.ETDataExtension; import com.exacttarget.fuelsdk.ETDataExtensionColumn; import com.exacttarget.fuelsdk.ETDataExtensionRow; -import com.exacttarget.fuelsdk.ETExpression; -import com.exacttarget.fuelsdk.ETFilter; import com.exacttarget.fuelsdk.ETResponse; import com.exacttarget.fuelsdk.ETResult; import com.exacttarget.fuelsdk.ETSdkException; @@ -42,7 +40,6 @@ import com.exacttarget.fuelsdk.internal.UpdateResult; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.etl.api.FailureCollector; -import io.cdap.plugin.sfmc.sink.MarketingCloudConf; import java.util.ArrayList; import java.util.Collection; @@ -86,7 +83,7 @@ public String getDataExtensionKey() { /** * Validate that the given schema is compatible with the given extension. * - * @param schema the schema to check compatibility with + * @param schema the schema to check compatibility with * @param collector failure collector * @throws ETSdkException if there was an error getting the column information for the data extension */ @@ -94,8 +91,7 @@ public void validateSchemaCompatibility(Schema schema, FailureCollector collecto call(() -> { Collection columns = getDataExtensionInfo().getColumnList(); if (columns == null || columns.isEmpty()) { - collector.addFailure(String.format("Data extension '%s' must exist.", - dataExtensionKey), null) + collector.addFailure(String.format("Data extension '%s' must exist.", dataExtensionKey), null) .withConfigProperty(MarketingCloudConf.DATA_EXTENSION); throw collector.getOrThrowException(); } @@ -111,8 +107,7 @@ public void validateSchemaCompatibility(Schema schema, FailureCollector collecto if (column.getIsRequired()) { collector.addFailure( String.format("Data extension '%s' contains a required column '%s' of type '%s' that is not present in " + - "the input schema.", dataExtensionKey, columnName, column.getType()) - , null) + "the input schema.", dataExtensionKey, columnName, column.getType()), null) .withConfigProperty(MarketingCloudConf.DATA_EXTENSION); } } else { @@ -134,7 +129,7 @@ public void validateSchemaCompatibility(Schema schema, FailureCollector collecto if (schemaType != Schema.Type.BOOLEAN) { collector.addFailure( String.format("Column '%s' is a boolean in data extension '%s', but is a '%s' in the input schema.", - originalName, dataExtensionKey, schemaTypeStr), + originalName, dataExtensionKey, schemaTypeStr), "Change the field schema to ensure it is a boolean or string type.") .withInputSchemaField(fieldName); } @@ -143,7 +138,7 @@ public void validateSchemaCompatibility(Schema schema, FailureCollector collecto if (fieldSchema.getLogicalType() != Schema.LogicalType.DECIMAL) { collector.addFailure( String.format("Column '%s' is a decimal in data extension '%s', but is a '%s' in the input schema.", - originalName, dataExtensionKey, schemaTypeStr), + originalName, dataExtensionKey, schemaTypeStr), "Change the field schema to ensure it is a decimal or string type.").withInputSchemaField(fieldName); } break; @@ -153,32 +148,29 @@ public void validateSchemaCompatibility(Schema schema, FailureCollector collecto case LOCALE: collector.addFailure( String.format("Column '%s' is a %s in data extension '%s', but is a '%s' in the input schema.", - originalName, column.getType().name().toLowerCase(), dataExtensionKey, - schemaTypeStr), - "Change the field schema to ensure it is a string type.") - .withInputSchemaField(fieldName); + originalName, column.getType().name().toLowerCase(), dataExtensionKey, schemaTypeStr), + "Change the field schema to ensure it is a string type.").withInputSchemaField(fieldName); break; case DATE: if (fieldSchema.getLogicalType() != Schema.LogicalType.DATE) { collector.addFailure( String.format("Column '%s' is a date in data extension '%s', but is a '%s' in the input schema.", - originalName, dataExtensionKey, schemaTypeStr), - "Change the field schema to ensure it is a date or string type." - ).withInputSchemaField(fieldName); + originalName, dataExtensionKey, schemaTypeStr), + "Change the field schema to ensure it is a date or string type.").withInputSchemaField(fieldName); } break; case NUMBER: if (schemaType != Schema.Type.INT) { collector.addFailure( String.format("Column '%s' is a number in data extension '%s', but is a '%s' in the input schema.", - originalName, dataExtensionKey, schemaTypeStr), + originalName, dataExtensionKey, schemaTypeStr), "Change the field schema to ensure it is an integer or string type.").withInputSchemaField(fieldName); } break; default: collector.addFailure( String.format("Unknown type '%s' for column '%s' in data extension '%s'.", - column.getType(), column.getName(), dataExtensionKey), + column.getType(), column.getName(), dataExtensionKey), "Supported types are: boolean, decimal, phone, text, email_address, locale, date and number.") .withInputSchemaField(fieldName); } @@ -199,31 +191,6 @@ public List scan() throws ETSdkException { return call(() -> ETDataExtension.select(client, "key=" + dataExtensionKey).getObjects()); } - public List pagedScan(Integer page, Integer pageSize) throws ETSdkException { - return call(() -> ETDataExtension.select(client, "key=" + dataExtensionKey, page, pageSize) - .getObjects()); - } - - public ETDataExtension retrieveDataExtension() throws ETSdkException { - //client.refreshToken(); - return call(() -> { - ETExpression expression = new ETExpression(); - expression.setProperty("DataExtension.CustomerKey"); - expression.setOperator(ETExpression.Operator.EQUALS); - expression.addValue(dataExtensionKey); - ETFilter filter = new ETFilter(); - filter.setExpression(expression); - ETResponse response = ETDataExtension.retrieve(client, ETDataExtension.class, (Integer) null, - (Integer) null, filter); - return response.getObject(); - }); - } - - public Integer fetchRecordCount() throws ETSdkException { - return call(() -> ETDataExtension.select(client, "key=" + dataExtensionKey, 1, 1) - .getTotalCount()); - } - public ETResponse insert(List rows) throws ETSdkException { return call(() -> create(client, rows)); } @@ -452,7 +419,7 @@ public static ETResponse update(ETClient client, externalObject = externalType.newInstance(); } catch (Exception ex) { throw new ETSdkException("could not instantiate " - + externalType.getName(), ex); + + externalType.getName(), ex); } externalObject.setClient(client); @@ -508,7 +475,7 @@ public static ETResponse update(ETClient client, * Used to run the client from the command line. Primarily used for testing purposes. * Expects arguments in the form [command] [key] [properties files] where properties file is expected * to contain lines like: - *

+ * * clientId=... * clientSecret=... * authEndpoint=... @@ -522,17 +489,12 @@ public static void main(String[] args) throws Exception { DataExtensionClient client = new DataExtensionClient(new ETClient(conf), key); - ETDataExtension dataExtension = client.retrieveDataExtension(); - System.out.println("Data Extension = " + dataExtension.getName()); - if ("describe".equals(command)) { Collection columns = client.getDataExtensionInfo().getColumnList(); - System.out.println(columns.stream().map(c -> c.getName() + " " + c.getType()) - .collect(Collectors.joining("\n"))); + System.out.println(columns.stream().map(c -> c.getName() + " " + c.getType()).collect(Collectors.joining("\n"))); } else if ("scan".equals(command)) { Collection columns = client.getDataExtensionInfo().getColumnList(); - List columnNames = columns.stream().map(ETDataExtensionColumn::getName) - .collect(Collectors.toList()); + List columnNames = columns.stream().map(ETDataExtensionColumn::getName).collect(Collectors.toList()); System.out.println(String.join(", ", columnNames)); for (ETDataExtensionRow row : client.scan()) { List values = new ArrayList<>(columnNames.size()); diff --git a/src/main/java/io/cdap/plugin/sfmc/common/DataExtensionInfo.java b/src/main/java/io/cdap/plugin/sfmc/sink/DataExtensionInfo.java similarity index 97% rename from src/main/java/io/cdap/plugin/sfmc/common/DataExtensionInfo.java rename to src/main/java/io/cdap/plugin/sfmc/sink/DataExtensionInfo.java index 277554a..a551df8 100644 --- a/src/main/java/io/cdap/plugin/sfmc/common/DataExtensionInfo.java +++ b/src/main/java/io/cdap/plugin/sfmc/sink/DataExtensionInfo.java @@ -14,7 +14,7 @@ * the License. */ -package io.cdap.plugin.sfmc.common; +package io.cdap.plugin.sfmc.sink; import com.exacttarget.fuelsdk.ETDataExtensionColumn; diff --git a/src/main/java/io/cdap/plugin/sfmc/sink/DataExtensionOutputFormat.java b/src/main/java/io/cdap/plugin/sfmc/sink/DataExtensionOutputFormat.java index a39df0b..7930b5f 100644 --- a/src/main/java/io/cdap/plugin/sfmc/sink/DataExtensionOutputFormat.java +++ b/src/main/java/io/cdap/plugin/sfmc/sink/DataExtensionOutputFormat.java @@ -18,7 +18,6 @@ import com.exacttarget.fuelsdk.ETSdkException; import io.cdap.cdap.api.data.format.StructuredRecord; -import io.cdap.plugin.sfmc.common.DataExtensionClient; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.JobContext; diff --git a/src/main/java/io/cdap/plugin/sfmc/sink/DataExtensionRecordWriter.java b/src/main/java/io/cdap/plugin/sfmc/sink/DataExtensionRecordWriter.java index 7eea8ca..76226ef 100644 --- a/src/main/java/io/cdap/plugin/sfmc/sink/DataExtensionRecordWriter.java +++ b/src/main/java/io/cdap/plugin/sfmc/sink/DataExtensionRecordWriter.java @@ -21,7 +21,6 @@ import com.exacttarget.fuelsdk.ETResult; import com.exacttarget.fuelsdk.ETSdkException; import io.cdap.cdap.api.data.format.StructuredRecord; -import io.cdap.plugin.sfmc.common.DataExtensionClient; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.RecordWriter; import org.apache.hadoop.mapreduce.TaskAttemptContext; diff --git a/src/main/java/io/cdap/plugin/sfmc/sink/MarketingCloudConf.java b/src/main/java/io/cdap/plugin/sfmc/sink/MarketingCloudConf.java index 0a65d30..a2ebedb 100644 --- a/src/main/java/io/cdap/plugin/sfmc/sink/MarketingCloudConf.java +++ b/src/main/java/io/cdap/plugin/sfmc/sink/MarketingCloudConf.java @@ -23,7 +23,6 @@ import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.api.plugin.PluginConfig; import io.cdap.cdap.etl.api.FailureCollector; -import io.cdap.plugin.sfmc.common.DataExtensionClient; import java.util.Collections; import java.util.HashMap; diff --git a/src/main/java/io/cdap/plugin/sfmc/sink/RecordDataExtensionRowConverter.java b/src/main/java/io/cdap/plugin/sfmc/sink/RecordDataExtensionRowConverter.java index ae27bea..0fefe6d 100644 --- a/src/main/java/io/cdap/plugin/sfmc/sink/RecordDataExtensionRowConverter.java +++ b/src/main/java/io/cdap/plugin/sfmc/sink/RecordDataExtensionRowConverter.java @@ -20,12 +20,13 @@ import com.exacttarget.fuelsdk.ETDataExtensionRow; import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; -import io.cdap.plugin.sfmc.common.DataExtensionInfo; import java.math.BigDecimal; import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.function.Predicate; /** * Converts StructuredRecords into DataExtensionRows. diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceClient.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceClient.java new file mode 100644 index 0000000..2f33c6f --- /dev/null +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceClient.java @@ -0,0 +1,165 @@ +/* + * Copyright © 2017-2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.sfmc.source; + +import com.exacttarget.fuelsdk.ETApiObject; +import com.exacttarget.fuelsdk.ETClient; +import com.exacttarget.fuelsdk.ETConfiguration; +import com.exacttarget.fuelsdk.ETDataExtension; +import com.exacttarget.fuelsdk.ETDataExtensionColumn; +import com.exacttarget.fuelsdk.ETDataExtensionRow; +import com.exacttarget.fuelsdk.ETExpression; +import com.exacttarget.fuelsdk.ETFilter; +import com.exacttarget.fuelsdk.ETResponse; +import com.exacttarget.fuelsdk.ETSdkException; +import io.cdap.plugin.sfmc.source.util.SalesforceColumn; +import io.cdap.plugin.sfmc.source.util.SalesforceObjectInfo; +import io.cdap.plugin.sfmc.source.util.SourceObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Utility class that provides methods to connect to Salesforce instance. + */ +public class SalesforceClient { + private static final Logger LOG = LoggerFactory.getLogger(SalesforceClient.class); + private final ETClient client; + + SalesforceClient(ETClient client) { + this.client = client; + } + + public static SalesforceClient create(String clientId, String clientSecret, String authEndpoint, + String soapEndpoint) throws ETSdkException { + ETConfiguration conf = new ETConfiguration(); + conf.set("clientId", clientId); + conf.set("clientSecret", clientSecret); + conf.set("authEndpoint", authEndpoint); + conf.set("soapEndpoint", soapEndpoint); + conf.set("useOAuth2Authentication", "true"); + ClassLoader oldCL = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(SalesforceClient.class.getClassLoader()); + return new SalesforceClient(new ETClient(conf)); + } finally { + Thread.currentThread().setContextClassLoader(oldCL); + } + } + + public List fetchObjectRecords(SourceObject object) { + try { + return fetchObjectData(client, object.getClassRef()); + } catch (ETSdkException e) { + return Collections.emptyList(); + } + } + + public SalesforceObjectInfo fetchObjectSchema(SourceObject object) { + return new SalesforceObjectInfo(object, fetchObjectFields(object.getClassRef()), 0); + } + + public List fetchDataExtensionRecords(String dataExtensionKey) throws ETSdkException { + return call(() -> { + ETExpression expression = buildDataExtensionExpression(dataExtensionKey); + + ETFilter filter = new ETFilter(); + filter.addProperty("key"); + filter.addProperty("name"); + filter.setExpression(expression); + + ETResponse response = client.retrieve(ETDataExtension.class, filter); + List extensionRows = response.getObjects(); + + if (extensionRows.isEmpty()) { + return Collections.emptyList(); + } + + ETDataExtension de = extensionRows.get(0); + return de.select().getObjects(); + }); + } + + public SalesforceObjectInfo fetchDataExtensionSchema(String dataExtensionKey) throws ETSdkException { + return call(() -> { + ETExpression expression = buildDataExtensionExpression(dataExtensionKey); + + ETFilter filter = new ETFilter(); + filter.setExpression(expression); + filter.addProperty("name"); + filter.addProperty("type"); + + ETResponse response = ETDataExtensionColumn.retrieve(client, ETDataExtensionColumn.class, + (Integer) null, (Integer) null, filter); + List columns = response.getObjects().stream() + .map(o -> new SalesforceColumn(o.getName(), o.getType().name())) + .collect(Collectors.toList()); + + return new SalesforceObjectInfo(SourceObject.DATA_EXTENSION, dataExtensionKey, columns, 0); + }); + } + + private ETExpression buildDataExtensionExpression(String dataExtensionKey) { + ETExpression expression = new ETExpression(); + + expression.setProperty("DataExtension.CustomerKey"); + expression.setOperator(ETExpression.Operator.EQUALS); + expression.addValue(dataExtensionKey); + + return expression; + } + + private List fetchObjectData(ETClient client, Class clazz) throws ETSdkException { + ETResponse etResponse = client.retrieve(clazz, new ETFilter()); + List rows = etResponse.getObjects(); + + return rows; + } + + private List fetchObjectFields(Class clazz) { + return Arrays.stream(clazz.getDeclaredFields()) + .map(o -> new SalesforceColumn(o.getName(), o.getType().getSimpleName().toUpperCase())) + .collect(Collectors.toList()); + } + + private T call(SFMCCall callable) throws ETSdkException { + ClassLoader oldClassloader = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); + return callable.call(); + } finally { + Thread.currentThread().setContextClassLoader(oldClassloader); + } + } + + /** + * A SFMC call + * + * @param type of return object + */ + private interface SFMCCall { + + /** + * Perform a call. + */ + T call() throws ETSdkException; + } +} diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java index 74ed0a9..a2829c7 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java @@ -16,14 +16,9 @@ package io.cdap.plugin.sfmc.source; -import com.exacttarget.fuelsdk.ETDataExtension; import io.cdap.cdap.api.data.format.StructuredRecord; -import io.cdap.cdap.api.data.schema.Schema; -import io.cdap.plugin.sfmc.common.DataExtensionClient; -import io.cdap.plugin.sfmc.common.DataExtensionInfo; -import io.cdap.plugin.sfmc.source.util.SalesforceColumn; import io.cdap.plugin.sfmc.source.util.SalesforceObjectInfo; -import io.cdap.plugin.sfmc.source.util.SchemaBuilder; +import io.cdap.plugin.sfmc.source.util.SourceObject; import io.cdap.plugin.sfmc.source.util.SourceQueryMode; import io.cdap.plugin.sfmc.source.util.Util; import org.apache.hadoop.conf.Configuration; @@ -40,7 +35,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.stream.Collectors; import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.MAX_PAGE_SIZE; @@ -63,8 +57,7 @@ public static List setInput(Configuration jobConfig, Sourc SalesforceJobConfiguration jobConf = new SalesforceJobConfiguration(jobConfig); jobConf.setPluginConfiguration(conf); - //Depending on conf value fetch the list of fields for each table and create schema object - //return the schema object for each table as SalesforceObjectInfo + //Depending on the selected objects in the conf, get the schema for each object as SalesforceObjectInfo List tableInfos = fetchTableInfo(mode, conf); jobConf.setTableInfos(tableInfos); @@ -73,64 +66,66 @@ public static List setInput(Configuration jobConfig, Sourc return tableInfos; } + /** + * Depending on conf value fetch the list of fields for each object and create schema object + * return the schema for each object as SalesforceObjectInfo + */ private static List fetchTableInfo(SourceQueryMode mode, SalesforceSourceConfig conf) { - //When mode = Table, fetch details from the table name provided in plugin config - if (mode == SourceQueryMode.SINGLE_OBJECT) { - SalesforceObjectInfo tableInfo = getTableMetaData(conf.getDataExtensionKey(), conf); - return (tableInfo == null) ? Collections.emptyList() : Collections.singletonList(tableInfo); - } + try { + SalesforceClient client = SalesforceClient.create(conf.getClientId(), conf.getClientSecret(), + conf.getAuthEndpoint(), conf.getSoapEndpoint()); - //When mode = Reporting, get the list of tables for application name provided in plugin config - //and then fetch details from each of the tables. - List tableInfos = new ArrayList<>(); + //When mode = SingleObject, fetch fields for the object selected in plugin config + if (mode == SourceQueryMode.SINGLE_OBJECT) { + SalesforceObjectInfo tableInfo = getTableMetaData(conf.getObject(), conf.getDataExtensionKey(), client); + return (tableInfo == null) ? Collections.emptyList() : Collections.singletonList(tableInfo); + } - List tableKeys = Util.splitToList(conf.getDataExtensionKeys(), ','); - for (String tableKey : tableKeys) { - SalesforceObjectInfo tableInfo = getTableMetaData(tableKey, conf); - if (tableInfo == null) { - continue; + //When mode = MultiObject, get the list of objects provided in plugin config and the fetch fields for each of + //then objects. + List tableInfos = new ArrayList<>(); + List objectList = conf.getObjectList(); + + for (SourceObject object : objectList) { + SalesforceObjectInfo tableInfo = null; + + if (object == SourceObject.DATA_EXTENSION) { + //if the object = Data Extension then get the list of data extension keys and then fetch fields for each of + //the data extension objects. + List dataExtensionKeys = Util.splitToList(conf.getDataExtensionKeys(), ','); + for (String dataExtensionKey : dataExtensionKeys) { + tableInfo = getTableMetaData(object, dataExtensionKey, client); + if (tableInfo == null) { + continue; + } + tableInfos.add(tableInfo); + } + } else { + tableInfo = getTableMetaData(object, "", client); + if (tableInfo != null) { + tableInfos.add(tableInfo); + } + } } - tableInfos.add(tableInfo); - } - return tableInfos; + return tableInfos; + } catch (Exception e) { + LOG.error("Error in fetchTableInfo()", e); + return Collections.emptyList(); + } } - private static SalesforceObjectInfo getTableMetaData(String tableKey, SalesforceSourceConfig conf) { - //Call API to fetch first record from the table - LOG.debug("getTableMetaData::tableKey = {}", tableKey); - String tableName = ""; - + /** + * Fetch the fields for passed object + */ + private static SalesforceObjectInfo getTableMetaData(SourceObject object, String dataExtensionKey, + SalesforceClient client) { try { - DataExtensionClient client = DataExtensionClient.create(tableKey, conf.getClientId(), - conf.getClientSecret(), conf.getAuthEndpoint(), conf.getSoapEndpoint()); - - ETDataExtension dataExtension = client.retrieveDataExtension(); - if (dataExtension == null) { - return null; + if (object == SourceObject.DATA_EXTENSION) { + return client.fetchDataExtensionSchema(dataExtensionKey); + } else { + return client.fetchObjectSchema(object); } - - tableName = dataExtension.getName(); - DataExtensionInfo metaData = client.getDataExtensionInfo(); - - List columns = metaData.getColumnList().stream() - .map(o -> new SalesforceColumn(o.getName(), o.getType().name())) - .collect(Collectors.toList()); - - LOG.debug("columns.size() = {}", columns.size()); - - if (columns == null || columns.isEmpty()) { - return null; - } - - SchemaBuilder schemaBuilder = new SchemaBuilder(); - //Schema schema = schemaBuilder.constructSchema(tableName, columns); - Schema schema = schemaBuilder.constructSchema(tableKey, columns); - - int totalRecords = client.fetchRecordCount(); - - LOG.debug("table {}, rows = {}", tableName, totalRecords); - return new SalesforceObjectInfo(tableKey, tableName, schema, totalRecords); } catch (Exception e) { LOG.error("Error in getTableMetaData()", e); return null; @@ -139,7 +134,6 @@ private static SalesforceObjectInfo getTableMetaData(String tableKey, Salesforce @Override public List getSplits(JobContext jobContext) throws IOException, InterruptedException { - LOG.debug("In getSplits()"); SalesforceJobConfiguration jobConfig = new SalesforceJobConfiguration(jobContext.getConfiguration()); SalesforceSourceConfig pluginConf = jobConfig.getPluginConf(); @@ -147,19 +141,9 @@ public List getSplits(JobContext jobContext) throws IOException, Int List resultSplits = new ArrayList<>(); for (SalesforceObjectInfo tableInfo : tableInfos) { - String tableKey = tableInfo.getTableKey(); + String tableKey = tableInfo.getObject().name(); String tableName = tableInfo.getTableName(); - int totalRecords = tableInfo.getRecordCount(); - if (totalRecords < MAX_PAGE_SIZE) { - //add single split for table and continue - resultSplits.add(new SalesforceInputSplit(tableKey, tableName, 1, totalRecords)); - continue; - } - - int pages = (tableInfo.getRecordCount() / MAX_PAGE_SIZE) + 1; - for (int page = 1; page <= pages; page++) { - resultSplits.add(new SalesforceInputSplit(tableKey, tableName, page, MAX_PAGE_SIZE)); - } + resultSplits.add(new SalesforceInputSplit(tableKey, tableName, 1, MAX_PAGE_SIZE)); } LOG.debug("# of split = {}", resultSplits.size()); @@ -171,8 +155,6 @@ public List getSplits(JobContext jobContext) throws IOException, Int public RecordReader createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { - LOG.debug("In createRecordReader()"); - SalesforceJobConfiguration jobConfig = new SalesforceJobConfiguration(taskAttemptContext.getConfiguration()); SalesforceSourceConfig pluginConf = jobConfig.getPluginConf(); diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputSplit.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputSplit.java index 51ec1ed..ad3874e 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputSplit.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputSplit.java @@ -27,7 +27,7 @@ * Salesforce input split */ public class SalesforceInputSplit extends InputSplit implements Writable { - private String tableKey; + private String objectName; private String tableName; private int page; private int length; @@ -36,15 +36,15 @@ public class SalesforceInputSplit extends InputSplit implements Writable { public SalesforceInputSplit() { } - public SalesforceInputSplit(String tableKey, String tableName, int page, int length) { - this.tableKey = tableKey; + public SalesforceInputSplit(String objectName, String tableName, int page, int length) { + this.objectName = objectName; this.tableName = tableName; this.page = page; this.length = length; } - public String getTableKey() { - return tableKey; + public String getObjectName() { + return objectName; } public String getTableName() { @@ -57,14 +57,14 @@ public int getPage() { @Override public void write(DataOutput dataOutput) throws IOException { - dataOutput.writeUTF(this.tableKey); + dataOutput.writeUTF(this.objectName); dataOutput.writeUTF(this.tableName); dataOutput.writeInt(this.page); } @Override public void readFields(DataInput dataInput) throws IOException { - this.tableKey = dataInput.readUTF(); + this.objectName = dataInput.readUTF(); this.tableName = dataInput.readUTF(); this.page = dataInput.readInt(); } diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java index 3d16ab3..b460688 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java @@ -16,14 +16,13 @@ package io.cdap.plugin.sfmc.source; +import com.exacttarget.fuelsdk.ETApiObject; import com.exacttarget.fuelsdk.ETDataExtensionRow; import com.google.common.base.Strings; import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; -import io.cdap.plugin.sfmc.common.DataExtensionClient; -import io.cdap.plugin.sfmc.common.DataExtensionInfo; -import io.cdap.plugin.sfmc.source.util.SalesforceColumn; -import io.cdap.plugin.sfmc.source.util.SchemaBuilder; +import io.cdap.plugin.sfmc.source.util.SalesforceObjectInfo; +import io.cdap.plugin.sfmc.source.util.SourceObject; import io.cdap.plugin.sfmc.source.util.SourceQueryMode; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.InputSplit; @@ -33,14 +32,14 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.lang.reflect.Method; import java.sql.SQLException; import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; import java.util.List; -import java.util.stream.Collectors; -import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.MAX_PAGE_SIZE; +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.DATA_EXTENSION_PREFIX; /** * Record reader that reads the entire contents of a Salesforce table. @@ -53,12 +52,13 @@ public class SalesforceRecordReader extends RecordReader tableFields; private Schema schema; - private String tableKey; + private SourceObject object; + private String dataExtensionKey = ""; private String tableName; private String tableNameField; - private List results; - private Iterator iterator; - private ETDataExtensionRow row; + private List results; + private Iterator iterator; + private ETApiObject row; SalesforceRecordReader(SalesforceSourceConfig pluginConf) { this.pluginConf = pluginConf; @@ -66,7 +66,6 @@ public class SalesforceRecordReader extends RecordReader schemaFields; try { - DataExtensionInfo metaData = client.getDataExtensionInfo(); - - List columns = metaData.getColumnList().stream() - .map(o -> new SalesforceColumn(o.getName(), o.getType().name())) - .collect(Collectors.toList()); + SalesforceObjectInfo metaData = null; + if (object == SourceObject.DATA_EXTENSION) { + metaData = client.fetchDataExtensionSchema(dataExtensionKey); + } else { + metaData = client.fetchObjectSchema(object); + } //Build schema - SchemaBuilder schemaBuilder = new SchemaBuilder(); - Schema tempSchema = schemaBuilder.constructSchema(tableName, columns); - tableFields = tempSchema.getFields(); + tableFields = metaData.getSchema().getFields(); schemaFields = new ArrayList<>(tableFields); if (pluginConf.getQueryMode() == SourceQueryMode.MULTI_OBJECT) { schemaFields.add(Schema.Field.of(tableNameField, Schema.of(Schema.Type.STRING))); } - } catch (Exception e) { schemaFields = Collections.emptyList(); } @@ -189,9 +187,53 @@ private void fetchSchema(DataExtensionClient client) { schema = Schema.recordOf(tableName, schemaFields); } - private Object convertToValue(String fieldName, Schema fieldSchema, ETDataExtensionRow record) { + /** + * Read data from ETApiObject and convert it to StructureRecord + */ + private void convertRecord(StructuredRecord.Builder recordBuilder, ETApiObject row) { + for (Schema.Field field : tableFields) { + String fieldName = field.getName(); + Object rawFieldValue = null; + if (row instanceof ETDataExtensionRow) { + rawFieldValue = ((ETDataExtensionRow) row).getColumn(fieldName); + } else { + rawFieldValue = getFieldValue(row, fieldName); + } + + Object fieldValue = convertToValue(fieldName, field.getSchema(), rawFieldValue); + + recordBuilder.set(fieldName, fieldValue); + } + } + + /** + * Read data from ETApiObject using reflection for a given field name + */ + private Object getFieldValue(ETApiObject row, String fieldName) { + try { + Method method = row.getClass().getMethod(createGetterName(fieldName)); + return method.invoke(row); + } catch (Exception e) { + LOG.error(String.format("Error while fetching %s.%s value", row.getClass().getSimpleName(), fieldName), e); + return null; + } + } + + /** + * Constructs the get method name to be used in reflection call + */ + private String createGetterName(String name) { + StringBuilder sb = new StringBuilder("get"); + sb.append(name.substring(0, 1).toUpperCase()); + sb.append(name.substring(1)); + return sb.toString(); + } + + /** + * Converts raw field value according to the schema field type + */ + private Object convertToValue(String fieldName, Schema fieldSchema, Object fieldValue) { Schema.Type fieldType = fieldSchema.getType(); - Object fieldValue = record.getColumn(fieldName); switch (fieldType) { case STRING: @@ -204,11 +246,11 @@ private Object convertToValue(String fieldName, Schema fieldSchema, ETDataExtens return convertToBooleanValue(fieldValue); case UNION: if (fieldSchema.isNullable()) { - return convertToValue(fieldName, fieldSchema.getNonNullable(), record); + return convertToValue(fieldName, fieldSchema.getNonNullable(), fieldValue); } throw new IllegalStateException( String.format("Field '%s' is of unexpected type '%s'. Declared 'complex UNION' types: %s", - fieldName, record.getColumn(fieldName).getClass().getSimpleName(), fieldSchema.getUnionSchemas())); + fieldName, fieldValue.getClass().getSimpleName(), fieldSchema.getUnionSchemas())); default: throw new IllegalStateException( String.format("Record type '%s' is not supported for field '%s'", fieldType.name(), fieldName)); diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java index 4edd390..914a6f1 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java @@ -90,36 +90,29 @@ public void prepareRun(BatchSourceContext context) throws Exception { Collection tables = SalesforceInputFormat.setInput(hConf, mode, conf); SettableArguments arguments = context.getArguments(); for (SalesforceObjectInfo tableInfo : tables) { - LOG.debug("add lineage for %s table", tableInfo.getTableName()); - arguments.set(TABLE_PREFIX + tableInfo.getTableKey(), tableInfo.getSchema().toString()); + arguments.set(TABLE_PREFIX + tableInfo.getTableName(), tableInfo.getSchema().toString()); recordLineage(context, tableInfo); } - LOG.debug("lineage added for %d tables", tables.size()); - context.setInput(Input.of(conf.getReferenceName(), new SourceInputFormatProvider(SalesforceInputFormat.class, hConf))); - - LOG.debug("end of prepareRun"); } @Override public void transform(KeyValue input, Emitter emitter) { - LOG.debug("In transform"); emitter.emit(input.getValue()); } private void recordLineage(BatchSourceContext context, SalesforceObjectInfo tableInfo) { - String tableKey = tableInfo.getTableKey(); String tableName = tableInfo.getTableName(); - String outputName = String.format("%s-%s", conf.getReferenceName(), tableKey); + String outputName = String.format("%s-%s", conf.getReferenceName(), tableName); Schema schema = tableInfo.getSchema(); LineageRecorder lineageRecorder = new LineageRecorder(context, outputName); lineageRecorder.createExternalDataset(schema); List fields = Objects.requireNonNull(schema).getFields(); if (fields != null && !fields.isEmpty()) { lineageRecorder.recordRead("Read", - String.format("Read from '%s (Key: %s)' Salesforce Data Extension.", tableName, tableKey), + String.format("Read from '%s' Salesforce object.", tableName), fields.stream().map(Schema.Field::getName).collect(Collectors.toList())); } } diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java index fbfe258..189f4bc 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java @@ -25,13 +25,13 @@ import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.plugin.common.Constants; import io.cdap.plugin.common.IdUtils; -import io.cdap.plugin.sfmc.common.DataExtensionClient; import io.cdap.plugin.sfmc.source.util.SourceObject; import io.cdap.plugin.sfmc.source.util.SourceQueryMode; import io.cdap.plugin.sfmc.source.util.Util; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; import java.util.List; import java.util.Optional; import javax.annotation.Nullable; @@ -68,37 +68,49 @@ public class SalesforceSourceConfig extends PluginConfig { @Name(PROPERTY_OBJECT_NAME) @Macro @Nullable - @Description("Specify the object for which data to be fetched. Note, this value will be ignored if the Mode is " + - "set to `Multi Object`.") + @Description("Specify the object for which data to be fetched. This can be one of following values: " + + "`Data Extension` - will allow user to fetch data for a single Data Extension object, " + + "`Campaign` - will allow user to fetch data for Campaign object, " + + "`Email` - will allow user to fetch data for Email object, " + + "`Mailing List` - will allow user to fetch data for Mailing List object. " + + "Note, this value will be ignored if the Mode is set to `Multi Object`.") private String objectName; @Name(PROPERTY_DATA_EXTENSION_KEY) @Macro @Nullable - @Description("Specify the data extension key from which data to be fetched. Note, this value will be ignored if " + - "the Mode is set to `Multi Object`.") + @Description("Specify the data extension key from which data to be fetched. Note, this value will be ignored in " + + "following two cases: 1. If the Mode is set to `Multi Object`, 2. If the selected object name is other than " + + "`Data Extension`.") private String dataExtensionKey; @Name(PROPERTY_OBJECT_LIST) @Macro @Nullable - @Description("Specify the comma-separated list of objects for which data to be fetched. Note, this value will be " + - "ignored if the Mode is set to `Single Object`.") + @Description("Specify the comma-separated list of objects for which data to be fetched; for example: " + + "'Object1,Object2'. This can be one or more values from following possible values: " + + "`Data Extension` - will allow user to fetch data for a single Data Extension object, " + + "`Campaign` - will allow user to fetch data for Campaign object, " + + "`Email` - will allow user to fetch data for Email object, " + + "`Mailing List` - will allow user to fetch data for Mailing List object. " + + "Note, this value will be ignored if the Mode is set to `Single Object`.") private String objectList; @Name(PROPERTY_DATA_EXTENSION_KEY_LIST) @Macro @Nullable @Description("Specify the data extension keys from which data to be fetched; for example: 'Key1,Key2'. " + - "Note, this value will be ignored if the Mode is set to `Single Object`.") + "Note, this value will be ignored in following two cases: 1. If the Mode is set to `Single Object`, " + + "2. If the selected object list does not contain `Data Extension` as one of the objects.") private String dataExtensionKeys; @Name(PROPERTY_TABLE_NAME_FIELD) @Macro @Nullable - @Description("The name of the field that holds the data extension name. Must not be the name of any data " + - "extension column that will be read. Defaults to `tablename`. Note, the Table name field value will be ignored " + - "if the Mode is set to `Single Object`.") + @Description("The name of the field that holds the object name to which the data belongs to. Must not be the name " + + "of any column for any of the objects that will be read. Defaults to `tablename`. In case of `Data Extension` " + + "object, this field will have value in `dataextension-[Data Extension Key]` format. Note, the Table name field " + + "value will be ignored if the Mode is set to `Single Object`.") private String tableNameField; @Name(PROPERTY_CLIENT_ID) @@ -160,7 +172,8 @@ public SourceQueryMode getQueryMode(FailureCollector collector) { collector.addFailure("Unsupported query mode value: " + queryMode, String.format("Supported modes are: %s", SourceQueryMode.getSupportedModes())) .withConfigProperty(PROPERTY_QUERY_MODE); - throw collector.getOrThrowException(); + collector.getOrThrowException(); + return null; } public SourceQueryMode getQueryMode() { @@ -169,9 +182,22 @@ public SourceQueryMode getQueryMode() { return sourceQueryMode.isPresent() ? sourceQueryMode.get() : null; } + public SourceObject getObject(FailureCollector collector) { + SourceObject sourceObject = getObject(); + if (sourceObject != null) { + return sourceObject; + } + + collector.addFailure("Unsupported object value: " + objectName, + String.format("Supported objects are: %s", SourceObject.getSupportedObjects())) + .withConfigProperty(PROPERTY_OBJECT_NAME); + collector.getOrThrowException(); + return null; + } + @Nullable - public String getObjectName() { - return objectName; + public SourceObject getObject() { + return getSourceObject(objectName); } @Nullable @@ -179,8 +205,40 @@ public String getDataExtensionKey() { return dataExtensionKey; } - public String getObjectList() { - return objectList; + public List getObjectList(FailureCollector collector) { + List objects = Util.splitToList(objectList, ','); + List sourceObjects = new ArrayList<>(); + + for (String object : objects) { + SourceObject sourceObject = getSourceObject(object); + if (sourceObject == null) { + collector.addFailure("Unsupported object value: " + object, + String.format("Supported objects are: %s", SourceObject.getSupportedObjects())) + .withConfigProperty(PROPERTY_OBJECT_LIST); + break; + } + + sourceObjects.add(sourceObject); + } + + return sourceObjects; + } + + @Nullable + public List getObjectList() { + List objects = Util.splitToList(objectList, ','); + List sourceObjects = new ArrayList<>(); + + for (String object : objects) { + SourceObject sourceObject = getSourceObject(object); + if (sourceObject == null) { + continue; + } + + sourceObjects.add(sourceObject); + } + + return sourceObjects; } @Nullable @@ -224,6 +282,12 @@ public void validate(FailureCollector collector) { validateQueryMode(collector); } + private SourceObject getSourceObject(String objectName) { + Optional sourceObject = SourceObject.fromValue(objectName); + + return sourceObject.isPresent() ? sourceObject.get() : null; + } + private void validateCredentials(FailureCollector collector) { if (!shouldConnect()) { return; @@ -261,7 +325,7 @@ private void validateCredentials(FailureCollector collector) { @VisibleForTesting void validateSalesforceConnection(FailureCollector collector) { try { - DataExtensionClient.create("", clientId, clientSecret, authEndpoint, soapEndpoint); + SalesforceClient.create(clientId, clientSecret, authEndpoint, soapEndpoint); } catch (ETSdkException e) { collector.addFailure("Unable to connect to Salesforce Instance.", "Ensure properties like Client ID, Client Secret, API Endpoint, Soap Endpoint, Auth Endpoint " + @@ -276,7 +340,7 @@ void validateSalesforceConnection(FailureCollector collector) { } private void validateQueryMode(FailureCollector collector) { - //according to query mode check if either table name/application exists or not + //according to query mode check if either object name / object list exists or not if (containsMacro(PROPERTY_QUERY_MODE)) { return; } @@ -296,29 +360,22 @@ private void validateMultiObjectQueryMode(FailureCollector collector) { return; } - List objects = Util.splitToList(getObjectList(), ','); + List objects = getObjectList(collector); + collector.getOrThrowException(); + if (objects.isEmpty()) { collector.addFailure("At least 1 Object must be specified.", null) .withConfigProperty(PROPERTY_OBJECT_LIST); } - for (String o : objects) { - Optional sourceObject = SourceObject.fromValue(o); - SourceObject object = sourceObject.isPresent() ? sourceObject.get() : null; - if (object == null) { - collector.addFailure("Unsupported object value: " + o, - String.format("Supported objects are: %s", SourceObject.getSupportedObjects())) - .withConfigProperty(PROPERTY_OBJECT_LIST); - throw collector.getOrThrowException(); + if (objects.contains(SourceObject.DATA_EXTENSION)) { + List dataExtensionKeyList = Util.splitToList(getDataExtensionKeys(), ','); + if (dataExtensionKeyList.isEmpty()) { + collector.addFailure("At least 1 Data Extension Key must be specified.", null) + .withConfigProperty(PROPERTY_DATA_EXTENSION_KEY_LIST); } } - List dataExtensionKeyList = Util.splitToList(getDataExtensionKeys(), ','); - if (dataExtensionKeyList.isEmpty()) { - collector.addFailure("At least 1 Data Extension Key must be specified.", null) - .withConfigProperty(PROPERTY_DATA_EXTENSION_KEY_LIST); - } - if (Util.isNullOrEmpty(tableNameField)) { collector.addFailure("Table name field must be specified.", null) .withConfigProperty(PROPERTY_TABLE_NAME_FIELD); @@ -330,21 +387,9 @@ private void validateSingleObjectQueryMode(FailureCollector collector) { return; } - if (Util.isNullOrEmpty(objectName)) { - collector.addFailure("Object Name must be specified.", null) - .withConfigProperty(PROPERTY_OBJECT_NAME); - } - - Optional sourceObject = SourceObject.fromValue(objectName); - SourceObject object = sourceObject.isPresent() ? sourceObject.get() : null; - if (object == null) { - collector.addFailure("Unsupported object value: " + objectName, - String.format("Supported objects are: %s", SourceObject.getSupportedObjects())) - .withConfigProperty(PROPERTY_OBJECT_NAME); - throw collector.getOrThrowException(); - } + SourceObject object = getObject(collector); - if (Util.isNullOrEmpty(dataExtensionKey)) { + if (object == SourceObject.DATA_EXTENSION && Util.isNullOrEmpty(dataExtensionKey)) { collector.addFailure("Data Extension Key must be specified.", null) .withConfigProperty(PROPERTY_DATA_EXTENSION_KEY); } diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceColumn.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceColumn.java index d9f6913..e87b8cc 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceColumn.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceColumn.java @@ -17,18 +17,13 @@ package io.cdap.plugin.sfmc.source.util; import com.google.common.base.Strings; -import com.google.gson.annotations.SerializedName; /** * Information about a Salesforce table column. */ public class SalesforceColumn { - @SerializedName("element") private String fieldName; - - @SerializedName("internal_type") private String typeName; - private String mandatory; public SalesforceColumn() { diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java index eea2397..2028287 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java @@ -24,7 +24,7 @@ public interface SalesforceConstants { /** * Salesforce plugin name. */ - String PLUGIN_NAME = "SalesforceDataExtension"; + String PLUGIN_NAME = "Salesforce"; /** * Configuration property name used to specify the query mode. @@ -83,5 +83,7 @@ public interface SalesforceConstants { String TABLE_PREFIX = "multisink."; + String DATA_EXTENSION_PREFIX = "dataextension-"; + int MAX_PAGE_SIZE = 2500; } diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceObjectInfo.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceObjectInfo.java index e485af9..f3cb247 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceObjectInfo.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceObjectInfo.java @@ -18,28 +18,42 @@ import io.cdap.cdap.api.data.schema.Schema; +import java.util.List; + +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.DATA_EXTENSION_PREFIX; + /** * Information about a Salesforce table. */ public class SalesforceObjectInfo { - private final String tableKey; - private final String tableName; + private final SourceObject object; + private final String dataExtensionKey; private final Schema schema; private final int recordCount; - public SalesforceObjectInfo(String tableKey, String tableName, Schema schema, int recordCount) { - this.tableKey = tableKey; - this.tableName = tableName; - this.schema = schema; + public SalesforceObjectInfo(SourceObject object, List columns, int recordCount) { + this(object, null, columns, recordCount); + } + + public SalesforceObjectInfo(SourceObject object, String dataExtensionKey, List columns, + int recordCount) { + this.object = object; + this.dataExtensionKey = dataExtensionKey; + SchemaBuilder schemaBuilder = new SchemaBuilder(); + this.schema = schemaBuilder.constructSchema(getTableName(), columns); this.recordCount = recordCount; } - public String getTableKey() { - return tableKey; + public SourceObject getObject() { + return object; } public String getTableName() { - return tableName; + if (getObject() == SourceObject.DATA_EXTENSION) { + return String.format("%s%s", DATA_EXTENSION_PREFIX, dataExtensionKey); + } else { + return getObject().getTableName(); + } } public Schema getSchema() { diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SchemaBuilder.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SchemaBuilder.java index 95b93ba..94878e2 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SchemaBuilder.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SchemaBuilder.java @@ -60,8 +60,10 @@ private Schema.Field transformToField(SalesforceColumn column) { private Schema createSchema(SalesforceColumn column) { switch (column.getTypeName().toLowerCase()) { case "decimal": + case "double": return Schema.of(Schema.Type.DOUBLE); case "number": + case "integer": return Schema.of(Schema.Type.INT); case "boolean": return Schema.of(Schema.Type.BOOLEAN); diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SourceObject.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SourceObject.java index d71d394..f6e41d2 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SourceObject.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SourceObject.java @@ -16,6 +16,12 @@ package io.cdap.plugin.sfmc.source.util; +import com.exacttarget.fuelsdk.ETApiObject; +import com.exacttarget.fuelsdk.ETCampaign; +import com.exacttarget.fuelsdk.ETDataExtensionRow; +import com.exacttarget.fuelsdk.ETEmail; +import com.exacttarget.fuelsdk.ETList; + import java.util.Arrays; import java.util.Optional; import java.util.stream.Collectors; @@ -29,12 +35,31 @@ public enum SourceObject { /** * This indicates data to be fetched from Data Extension */ - DATA_EXTENSION("Data Extension"); + DATA_EXTENSION("Data Extension", "dataextension", ETDataExtensionRow.class), + + /** + * This indicates data to be fetched from Campaign + */ + CAMPAIGN("Campaign", "campaign", ETCampaign.class), + + /** + * This indicates data to be fetched from Email + */ + EMAIL("Email", "email", ETEmail.class), + + /** + * This indicates data to be fetched from Mailing List + */ + MAILING_LIST("Mailing List", "mailinglist", ETList.class); private final String value; + private final String tableName; + private final Class classRef; - SourceObject(String value) { + SourceObject(String value, String tableName, Class classRef) { this.value = value; + this.tableName = tableName; + this.classRef = classRef; } /** @@ -57,4 +82,12 @@ public static String getSupportedObjects() { public String getValue() { return value; } + + public String getTableName() { + return tableName; + } + + public Class getClassRef() { + return classRef; + } } diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SourceQueryMode.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SourceQueryMode.java index 07d8b4e..7dfee4f 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SourceQueryMode.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SourceQueryMode.java @@ -27,12 +27,12 @@ public enum SourceQueryMode { /** - * Mode will be used to fetch data for multiple data extensions + * Mode will be used to fetch data for multiple objects */ MULTI_OBJECT("Multi Object"), /** - * Mode will be used to fetch data for single data extension + * Mode will be used to fetch data for single object */ SINGLE_OBJECT("Single Object"); diff --git a/widgets/SalesforceDataExtension-batchsource.json b/widgets/Salesforce-batchsource.json similarity index 89% rename from widgets/SalesforceDataExtension-batchsource.json rename to widgets/Salesforce-batchsource.json index dee9fcb..e8aa2c2 100644 --- a/widgets/SalesforceDataExtension-batchsource.json +++ b/widgets/Salesforce-batchsource.json @@ -38,7 +38,10 @@ "name": "objectName", "widget-attributes": { "values": [ - "Data Extension" + "Data Extension", + "Campaign", + "Email", + "Mailing List" ], "default": "Data Extension" } @@ -65,12 +68,22 @@ { "id": "Data Extension", "label": "Data Extension" + }, + { + "id": "Campaign", + "label": "Campaign" + }, + { + "id": "Email", + "label": "Email" + }, + { + "id": "Mailing List", + "label": "Mailing List" } ], "delimiter": ",", - "default": [ - "Data Extension" - ] + "default": "Data Extension" } }, { From 490ef64825592c85e5fdef4f7750a19904b29723 Mon Sep 17 00:00:00 2001 From: amolkokane76 Date: Tue, 14 Apr 2020 13:23:37 +0530 Subject: [PATCH 13/19] Added filters section to widget json to hide/show configs --- ...alesforce_source_multiobject_pipeline.json | 3 +- ...lesforce_source_singleobject_pipeline.json | 3 +- .../sfmc/source/SalesforceInputFormat.java | 27 +++++--- .../sfmc/source/SalesforceRecordReader.java | 28 ++++++-- .../plugin/sfmc/source/SalesforceSource.java | 15 +++++ .../sfmc/source/SalesforceSourceConfig.java | 16 ++++- .../sfmc/source/util/SalesforceConstants.java | 5 ++ .../source/SalesforceSourceConfigHelper.java | 3 +- widgets/Salesforce-batchsource.json | 64 ++++++++++++++++++- 9 files changed, 145 insertions(+), 19 deletions(-) diff --git a/examples/example_salesforce_source_multiobject_pipeline.json b/examples/example_salesforce_source_multiobject_pipeline.json index b539f29..b15ae06 100644 --- a/examples/example_salesforce_source_multiobject_pipeline.json +++ b/examples/example_salesforce_source_multiobject_pipeline.json @@ -50,7 +50,8 @@ "restEndpoint": "${REST_ENDPOINT}", "dataExtensionKeyList": "${DATA_EXTENSION_KEY_LIST}", "objectName": "Data Extension", - "objectList": "${SFMC_OBJECT_LIST}" + "objectList": "${SFMC_OBJECT_LIST}", + "failOnError": "false" } }, "outputSchema": [ diff --git a/examples/example_salesforce_source_singleobject_pipeline.json b/examples/example_salesforce_source_singleobject_pipeline.json index 2c4fd39..522546c 100644 --- a/examples/example_salesforce_source_singleobject_pipeline.json +++ b/examples/example_salesforce_source_singleobject_pipeline.json @@ -50,7 +50,8 @@ "soapEndpoint": "${SOAP_ENDPOINT}", "restEndpoint": "${REST_ENDPOINT}", "objectName": "${SFMC_OBJECT_NAME}", - "objectList": "Data Extension" + "objectList": "Data Extension", + "failOnError": "false" } }, "outputSchema": [ diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java index a2829c7..6d358aa 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java @@ -48,8 +48,8 @@ public class SalesforceInputFormat extends InputFormat setInput(Configuration jobConfig, SourceQueryMode mode, @@ -75,9 +75,12 @@ private static List fetchTableInfo(SourceQueryMode mode, S SalesforceClient client = SalesforceClient.create(conf.getClientId(), conf.getClientSecret(), conf.getAuthEndpoint(), conf.getSoapEndpoint()); + boolean failOnError = conf.isFailOnError(); + //When mode = SingleObject, fetch fields for the object selected in plugin config if (mode == SourceQueryMode.SINGLE_OBJECT) { - SalesforceObjectInfo tableInfo = getTableMetaData(conf.getObject(), conf.getDataExtensionKey(), client); + SalesforceObjectInfo tableInfo = getTableMetaData(conf.getObject(), conf.getDataExtensionKey(), client, + failOnError); return (tableInfo == null) ? Collections.emptyList() : Collections.singletonList(tableInfo); } @@ -94,14 +97,14 @@ private static List fetchTableInfo(SourceQueryMode mode, S //the data extension objects. List dataExtensionKeys = Util.splitToList(conf.getDataExtensionKeys(), ','); for (String dataExtensionKey : dataExtensionKeys) { - tableInfo = getTableMetaData(object, dataExtensionKey, client); + tableInfo = getTableMetaData(object, dataExtensionKey, client, failOnError); if (tableInfo == null) { continue; } tableInfos.add(tableInfo); } } else { - tableInfo = getTableMetaData(object, "", client); + tableInfo = getTableMetaData(object, "", client, failOnError); if (tableInfo != null) { tableInfos.add(tableInfo); } @@ -110,7 +113,11 @@ private static List fetchTableInfo(SourceQueryMode mode, S return tableInfos; } catch (Exception e) { - LOG.error("Error in fetchTableInfo()", e); + if (conf.isFailOnError()) { + LOG.error("Error in fetchTableInfo()", e); + } else { + LOG.warn("Failed in fetchTableInfo()", e); + } return Collections.emptyList(); } } @@ -119,7 +126,7 @@ private static List fetchTableInfo(SourceQueryMode mode, S * Fetch the fields for passed object */ private static SalesforceObjectInfo getTableMetaData(SourceObject object, String dataExtensionKey, - SalesforceClient client) { + SalesforceClient client, boolean failOnError) { try { if (object == SourceObject.DATA_EXTENSION) { return client.fetchDataExtensionSchema(dataExtensionKey); @@ -127,7 +134,11 @@ private static SalesforceObjectInfo getTableMetaData(SourceObject object, String return client.fetchObjectSchema(object); } } catch (Exception e) { - LOG.error("Error in getTableMetaData()", e); + if (failOnError) { + LOG.error("Error in getTableMetaData()", e); + } else { + LOG.warn("Failed in getTableMetaData()", e); + } return null; } } diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java index b460688..f925a4f 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java @@ -85,8 +85,12 @@ public boolean nextKeyValue() throws IOException { pos++; } catch (Exception e) { - LOG.error("Error in nextKeyValue", e); - throw new IOException("Exception in nextKeyValue", e); + if (pluginConf.isFailOnError()) { + LOG.error("Error in nextKeyValue", e); + throw new IOException("Exception in nextKeyValue", e); + } else { + LOG.warn("Failed in nextKeyValue", e); + } } return true; } @@ -109,8 +113,12 @@ public StructuredRecord getCurrentValue() throws IOException { convertRecord(recordBuilder, row); } catch (Exception e) { - LOG.error("Error decoding row from table " + tableName, e); - throw new IOException("Error decoding row from table " + tableName, e); + if (pluginConf.isFailOnError()) { + LOG.error("Error decoding row from table " + tableName, e); + throw new IOException("Error decoding row from table " + tableName, e); + } else { + LOG.warn("Failed decoding row from table " + tableName, e); + } } return recordBuilder.build(); } @@ -155,7 +163,11 @@ private void fetchData() { } } catch (Exception e) { results = Collections.emptyList(); - LOG.error("Error while fetching data", e); + if (pluginConf.isFailOnError()) { + LOG.error("Error while fetching data", e); + } else { + LOG.warn("Failed while fetching data", e); + } } iterator = results.iterator(); @@ -214,7 +226,11 @@ private Object getFieldValue(ETApiObject row, String fieldName) { Method method = row.getClass().getMethod(createGetterName(fieldName)); return method.invoke(row); } catch (Exception e) { - LOG.error(String.format("Error while fetching %s.%s value", row.getClass().getSimpleName(), fieldName), e); + if (pluginConf.isFailOnError()) { + LOG.error(String.format("Error while fetching %s.%s value", row.getClass().getSimpleName(), fieldName), e); + } else { + LOG.warn(String.format("Failed while fetching %s.%s value", row.getClass().getSimpleName(), fieldName), e); + } return null; } } diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java index 914a6f1..f9dc6db 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java @@ -24,6 +24,7 @@ import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.api.dataset.lib.KeyValue; import io.cdap.cdap.etl.api.Emitter; +import io.cdap.cdap.etl.api.Engine; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.PipelineConfigurer; import io.cdap.cdap.etl.api.StageConfigurer; @@ -40,7 +41,10 @@ import org.slf4j.LoggerFactory; import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; @@ -76,6 +80,17 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) { // Since we have validated all the properties, throw an exception if there are any errors in the collector. // This is to avoid adding same validation errors again in getSchema method call collector.getOrThrowException(); + + if (conf.isFailOnError()) { + if (pipelineConfigurer.getEngine() == Engine.SPARK) { + pipelineConfigurer.setPipelineProperties(Collections.singletonMap("spark.task.maxFailures", "1")); + } else if (pipelineConfigurer.getEngine() == Engine.MAPREDUCE) { + Map properties = new HashMap<>(); + properties.put("mapreduce.reduce.maxattempts", "1"); + properties.put("mapreduce.map.maxattempts", "1"); + pipelineConfigurer.setPipelineProperties(properties); + } + } } @Override diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java index 189f4bc..b9536cf 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java @@ -42,6 +42,7 @@ import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_CLIENT_SECRET; import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_DATA_EXTENSION_KEY; import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_DATA_EXTENSION_KEY_LIST; +import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_FAIL_ON_ERROR; import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_OBJECT_LIST; import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_OBJECT_NAME; import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_QUERY_MODE; @@ -141,10 +142,17 @@ public class SalesforceSourceConfig extends PluginConfig { "For example, https://instance.soap.marketingcloudapis.com/Service.asmx") private String soapEndpoint; + @Macro + @Nullable + @Name(PROPERTY_FAIL_ON_ERROR) + @Description("Whether to fail the pipeline if it fails while reading.") + private Boolean failOnError; + public SalesforceSourceConfig(String referenceName, String queryMode, @Nullable String objectName, @Nullable String dataExtensionKey, @Nullable String objectList, @Nullable String dataExtensionKeys, @Nullable String tableNameField, String clientId, - String clientSecret, String restEndpoint, String authEndpoint, String soapEndpoint) { + String clientSecret, String restEndpoint, String authEndpoint, String soapEndpoint, + Boolean failOnError) { this.referenceName = referenceName; this.queryMode = queryMode; this.objectName = objectName; @@ -157,6 +165,7 @@ public SalesforceSourceConfig(String referenceName, String queryMode, @Nullable this.restEndpoint = restEndpoint; this.authEndpoint = authEndpoint; this.soapEndpoint = soapEndpoint; + this.failOnError = failOnError; } public String getReferenceName() { @@ -271,6 +280,11 @@ public String getSoapEndpoint() { return soapEndpoint; } + @Nullable + public Boolean isFailOnError() { + return failOnError == null ? false : failOnError; + } + /** * Validates {@link SalesforceSourceConfig} instance. */ diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java index 2028287..b8f3b3f 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java @@ -81,6 +81,11 @@ public interface SalesforceConstants { */ String PROPERTY_SOAP_API_ENDPOINT = "soapEndpoint"; + /** + * Configuration property name used to specify Fail on Error flag + */ + String PROPERTY_FAIL_ON_ERROR = "failOnError"; + String TABLE_PREFIX = "multisink."; String DATA_EXTENSION_PREFIX = "dataextension-"; diff --git a/src/test/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfigHelper.java b/src/test/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfigHelper.java index 0ca4aff..2317cbb 100644 --- a/src/test/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfigHelper.java +++ b/src/test/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfigHelper.java @@ -110,7 +110,8 @@ public ConfigBuilder setSoapEndpoint(String soapEndpoint) { public SalesforceSourceConfig build() { return new SalesforceSourceConfig(referenceName, queryMode, objectName, dataExtensionKey, objectList, - dataExtensionKeys, tableNameField, clientId, clientSecret, restEndpoint, authEndpoint, soapEndpoint); + dataExtensionKeys, tableNameField, clientId, clientSecret, restEndpoint, authEndpoint, soapEndpoint, + false); } } } diff --git a/widgets/Salesforce-batchsource.json b/widgets/Salesforce-batchsource.json index e8aa2c2..43b88db 100644 --- a/widgets/Salesforce-batchsource.json +++ b/widgets/Salesforce-batchsource.json @@ -149,6 +149,30 @@ } } ] + }, + { + "label": "Advanced", + "properties": [ + { + "widget-type": "radio-group", + "name" : "failOnError", + "label" : "Error handling", + "widget-attributes": { + "layout": "inline", + "default": "false", + "options": [ + { + "id": "true", + "label": "Stop on error" + }, + { + "id": "false", + "label": "Skip on error" + } + ] + } + } + ] } ], "outputs": [ ], @@ -158,5 +182,43 @@ "ref-property-name": "referenceName" } ] - } + }, + "filters": [ + { + "name": "showSingleObject", + "condition": { + "expression": "queryMode == 'Single Object'" + }, + "show": [ + { + "type": "property", + "name": "objectName" + }, + { + "type": "property", + "name": "dataExtensionKey" + } + ] + }, + { + "name": "showMultiObject", + "condition": { + "expression": "queryMode == 'Multi Object'" + }, + "show": [ + { + "type": "property", + "name": "objectList" + }, + { + "type": "property", + "name": "dataExtensionKeyList" + }, + { + "type": "property", + "name": "tableNameField" + } + ] + } + ] } From a386c648860aeac297f5938c0f6d5452e4955419 Mon Sep 17 00:00:00 2001 From: amolkokane76 Date: Fri, 17 Apr 2020 15:31:22 +0530 Subject: [PATCH 14/19] Updated readme file and added missing javadocs --- docs/Salesforce-batchsource.md | 9 +++ .../plugin/sfmc/source/SalesforceClient.java | 55 +++++++++++++++++-- .../sfmc/source/SalesforceInputFormat.java | 11 ++-- .../sfmc/source/SalesforceInputSplit.java | 10 +++- .../sfmc/source/SalesforceRecordReader.java | 8 +-- .../plugin/sfmc/source/SalesforceSource.java | 5 +- .../sfmc/source/SalesforceSourceConfig.java | 45 +++++++++++++++ .../sfmc/source/util/SalesforceColumn.java | 6 ++ .../sfmc/source/util/SalesforceConstants.java | 18 +++--- .../source/util/SalesforceObjectInfo.java | 20 +++++++ .../sfmc/source/util/SchemaBuilder.java | 9 ++- .../plugin/sfmc/source/util/SourceObject.java | 8 +-- .../sfmc/source/util/SourceQueryMode.java | 4 +- .../io/cdap/plugin/sfmc/source/util/Util.java | 13 +++++ widgets/Salesforce-batchsource.json | 12 ++++ 15 files changed, 200 insertions(+), 33 deletions(-) diff --git a/docs/Salesforce-batchsource.md b/docs/Salesforce-batchsource.md index bc32bef..55ecd59 100644 --- a/docs/Salesforce-batchsource.md +++ b/docs/Salesforce-batchsource.md @@ -76,3 +76,12 @@ For example, `https://instance.auth.marketingcloudapis.com/` **REST API Base URI**: The REST API Base URL associated for the Server-to-Server API integration. For example, `https://instance.rest.marketingcloudapis.com/` + +**Error handling:** Whether to fail the pipeline if an error encountered while reading records. This can be one of +following values: + +`Stop on error` - will fail the pipeline if an error encountered, + +`Skip on error` - will not fail the pipeline even if an error encountered and will continue the pipeline. + +Note: the default value is `Skip on error`. \ No newline at end of file diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceClient.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceClient.java index 2f33c6f..fb73991 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceClient.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceClient.java @@ -48,6 +48,16 @@ public class SalesforceClient { this.client = client; } + /** + * Initialize the connection with Salesforce Marketing Cloud using FuelSDK. + * + * @param clientId The Salesforce Marketing Cloud Client Id + * @param clientSecret The Salesforce Marketing Cloud Client Secret + * @param authEndpoint Auth Endpoint url for Salesforce Marketing Cloud + * @param soapEndpoint SOAP Endpoint url for Salesforce Marketing Cloud + * @return The instance of SalesforceClient object + * @throws ETSdkException The FuelSDKException + */ public static SalesforceClient create(String clientId, String clientSecret, String authEndpoint, String soapEndpoint) throws ETSdkException { ETConfiguration conf = new ETConfiguration(); @@ -65,6 +75,13 @@ public static SalesforceClient create(String clientId, String clientSecret, Stri } } + /** + * Fetch records for passed object from Salesforce Marketing Cloud. + * + * @param object The SourceObject which tells what data to be fetched from Salesforce + * Marketing Cloud + * @return The list of ETApiObject representing the records from requested object + */ public List fetchObjectRecords(SourceObject object) { try { return fetchObjectData(client, object.getClassRef()); @@ -73,11 +90,27 @@ public List fetchObjectRecords(SourceObject object) { } } + /** + * Fetch the schema information for passed object from Salesforce Marketing Cloud. + * + * @param object The SourceObject which tells for which schema to be fetched from Salesforce + * Marketing Cloud + * @return The instance of SalesforceObjectInfo object + */ public SalesforceObjectInfo fetchObjectSchema(SourceObject object) { return new SalesforceObjectInfo(object, fetchObjectFields(object.getClassRef()), 0); } - public List fetchDataExtensionRecords(String dataExtensionKey) throws ETSdkException { + /** + * Fetch records for passed object from Salesforce Marketing Cloud. + * + * @param dataExtensionKey The data extension key for which data to be fetched from Salesforce + * Marketing Cloud + * @return The list of ETDataExtensionRow representing the records from requested data extension + * @throws ETSdkException The FuelSDKException + */ + public List fetchDataExtensionRecords(String dataExtensionKey) + throws ETSdkException { return call(() -> { ETExpression expression = buildDataExtensionExpression(dataExtensionKey); @@ -98,7 +131,16 @@ public List fetchDataExtensionRecords(String dataExtensionKe }); } - public SalesforceObjectInfo fetchDataExtensionSchema(String dataExtensionKey) throws ETSdkException { + /** + * Fetch the schema information for passed object from Salesforce Marketing Cloud. + * + * @param dataExtensionKey The data extension key for which schema to be fetched from + * Salesforce Marketing Cloud + * @return The instance of SalesforceObjectInfo object + * @throws ETSdkException The FuelSDKException + */ + public SalesforceObjectInfo fetchDataExtensionSchema(String dataExtensionKey) + throws ETSdkException { return call(() -> { ETExpression expression = buildDataExtensionExpression(dataExtensionKey); @@ -107,8 +149,8 @@ public SalesforceObjectInfo fetchDataExtensionSchema(String dataExtensionKey) th filter.addProperty("name"); filter.addProperty("type"); - ETResponse response = ETDataExtensionColumn.retrieve(client, ETDataExtensionColumn.class, - (Integer) null, (Integer) null, filter); + ETResponse response = ETDataExtensionColumn.retrieve(client, + ETDataExtensionColumn.class, (Integer) null, (Integer) null, filter); List columns = response.getObjects().stream() .map(o -> new SalesforceColumn(o.getName(), o.getType().name())) .collect(Collectors.toList()); @@ -127,7 +169,8 @@ private ETExpression buildDataExtensionExpression(String dataExtensionKey) { return expression; } - private List fetchObjectData(ETClient client, Class clazz) throws ETSdkException { + private List fetchObjectData(ETClient client, Class clazz) + throws ETSdkException { ETResponse etResponse = client.retrieve(clazz, new ETFilter()); List rows = etResponse.getObjects(); @@ -151,7 +194,7 @@ private T call(SFMCCall callable) throws ETSdkException { } /** - * A SFMC call + * A SFMC call. * * @param type of return object */ diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java index 6d358aa..c4e2bcf 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java @@ -39,7 +39,7 @@ import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.MAX_PAGE_SIZE; /** - * Salesforce input format + * Salesforce input format. */ public class SalesforceInputFormat extends InputFormat { private static final Logger LOG = LoggerFactory.getLogger(SalesforceInputFormat.class); @@ -67,8 +67,11 @@ public static List setInput(Configuration jobConfig, Sourc } /** - * Depending on conf value fetch the list of fields for each object and create schema object - * return the schema for each object as SalesforceObjectInfo + * Depending on conf value fetch the list of fields for each object and create schema object. + * + * @param mode the query mode + * @param conf the plugin conf + * @return Collection of SalesforceObjectInfo containing table and schema. */ private static List fetchTableInfo(SourceQueryMode mode, SalesforceSourceConfig conf) { try { @@ -123,7 +126,7 @@ private static List fetchTableInfo(SourceQueryMode mode, S } /** - * Fetch the fields for passed object + * Fetch the fields for passed object. */ private static SalesforceObjectInfo getTableMetaData(SourceObject object, String dataExtensionKey, SalesforceClient client, boolean failOnError) { diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputSplit.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputSplit.java index ad3874e..9d146fe 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputSplit.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputSplit.java @@ -24,7 +24,7 @@ import java.io.IOException; /** - * Salesforce input split + * Salesforce input split. */ public class SalesforceInputSplit extends InputSplit implements Writable { private String objectName; @@ -36,6 +36,14 @@ public class SalesforceInputSplit extends InputSplit implements Writable { public SalesforceInputSplit() { } + /** + * Constructor for SalesforceInputSplit. + * + * @param objectName The object name + * @param tableName The corresponding table name + * @param page The page index + * @param length The page size + */ public SalesforceInputSplit(String objectName, String tableName, int page, int length) { this.objectName = objectName; this.tableName = tableName; diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java index f925a4f..5e87d8d 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java @@ -200,7 +200,7 @@ private void fetchSchema(SalesforceClient client) { } /** - * Read data from ETApiObject and convert it to StructureRecord + * Read data from ETApiObject and convert it to StructureRecord. */ private void convertRecord(StructuredRecord.Builder recordBuilder, ETApiObject row) { for (Schema.Field field : tableFields) { @@ -219,7 +219,7 @@ private void convertRecord(StructuredRecord.Builder recordBuilder, ETApiObject r } /** - * Read data from ETApiObject using reflection for a given field name + * Read data from ETApiObject using reflection for a given field name. */ private Object getFieldValue(ETApiObject row, String fieldName) { try { @@ -236,7 +236,7 @@ private Object getFieldValue(ETApiObject row, String fieldName) { } /** - * Constructs the get method name to be used in reflection call + * Constructs the get method name to be used in reflection call. */ private String createGetterName(String name) { StringBuilder sb = new StringBuilder("get"); @@ -246,7 +246,7 @@ private String createGetterName(String name) { } /** - * Converts raw field value according to the schema field type + * Converts raw field value according to the schema field type. */ private Object convertToValue(String fieldName, Schema fieldSchema, Object fieldValue) { Schema.Type fieldType = fieldSchema.getType(); diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java index f9dc6db..230523a 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java @@ -77,8 +77,9 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) { FailureCollector collector = stageConfigurer.getFailureCollector(); conf.validate(collector); - // Since we have validated all the properties, throw an exception if there are any errors in the collector. - // This is to avoid adding same validation errors again in getSchema method call + // Since we have validated all the properties, throw an exception if there are any + // errors in the collector. This is to avoid adding same validation errors again in + // getSchema method call collector.getOrThrowException(); if (conf.isFailOnError()) { diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java index b9536cf..5ed773b 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java @@ -148,6 +148,23 @@ public class SalesforceSourceConfig extends PluginConfig { @Description("Whether to fail the pipeline if it fails while reading.") private Boolean failOnError; + /** + * Constructor for SalesforceSourceConfig object. + * + * @param referenceName The reference name + * @param queryMode The query mode + * @param objectName The object name to be fetched from Salesforce Marketing Cloud + * @param dataExtensionKey The data extension key to be fetched from Salesforce Marketing Cloud + * @param objectList The list of objects to be fetched from Salesforce Marketing Cloud + * @param dataExtensionKeys The list of data extension keys to be fetched from Salesforce Marketing Cloud + * @param tableNameField The field name to hold the table name value + * @param clientId The Salesforce Marketing Cloud Client Id + * @param clientSecret The Salesforce Marketing Cloud Client Secret + * @param restEndpoint The REST API endpoint for Salesforce Marketing Cloud + * @param authEndpoint The AUTH API endpoint for Salesforce Marketing Cloud + * @param soapEndpoint The SOAP API endpoint for Salesforce Marketing Cloud + * @param failOnError A flag to decide what is to be done to the pipeline execution in case of errors + */ public SalesforceSourceConfig(String referenceName, String queryMode, @Nullable String objectName, @Nullable String dataExtensionKey, @Nullable String objectList, @Nullable String dataExtensionKeys, @Nullable String tableNameField, String clientId, @@ -172,6 +189,12 @@ public String getReferenceName() { return referenceName; } + /** + * Returns the query mode chosen. + * + * @param collector The failure collector to collect the errors + * @return An instance of SourceQueryMode + */ public SourceQueryMode getQueryMode(FailureCollector collector) { SourceQueryMode mode = getQueryMode(); if (mode != null) { @@ -185,12 +208,23 @@ public SourceQueryMode getQueryMode(FailureCollector collector) { return null; } + /** + * Returns the query mode chosen. + * + * @return An instance of SourceQueryMode + */ public SourceQueryMode getQueryMode() { Optional sourceQueryMode = SourceQueryMode.fromValue(queryMode); return sourceQueryMode.isPresent() ? sourceQueryMode.get() : null; } + /** + * Returns selected object. + * + * @param collector The failure collector to collect the errors + * @return An instance of SourceObject + */ public SourceObject getObject(FailureCollector collector) { SourceObject sourceObject = getObject(); if (sourceObject != null) { @@ -214,6 +248,12 @@ public String getDataExtensionKey() { return dataExtensionKey; } + /** + * Returns list of selected objects. + * + * @param collector The failure collector to collect the errors + * @return The list of SourceObject + */ public List getObjectList(FailureCollector collector) { List objects = Util.splitToList(objectList, ','); List sourceObjects = new ArrayList<>(); @@ -233,6 +273,11 @@ public List getObjectList(FailureCollector collector) { return sourceObjects; } + /** + * Returns list of selected objects. + * + * @return The list of SourceObject + */ @Nullable public List getObjectList() { List objects = Util.splitToList(objectList, ','); diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceColumn.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceColumn.java index e87b8cc..7c3aa67 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceColumn.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceColumn.java @@ -29,6 +29,12 @@ public class SalesforceColumn { public SalesforceColumn() { } + /** + * Constructor for SalesforceColumn. + * + * @param fieldName The column name + * @param typeName The data type name + */ public SalesforceColumn(String fieldName, String typeName) { this.fieldName = fieldName; this.typeName = typeName; diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java index b8f3b3f..c236956 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java @@ -32,17 +32,17 @@ public interface SalesforceConstants { String PROPERTY_QUERY_MODE = "queryMode"; /** - * Configuration property name used to specify the object name + * Configuration property name used to specify the object name. */ String PROPERTY_OBJECT_NAME = "objectName"; /** - * Configuration property name used to specify data extension key + * Configuration property name used to specify data extension key. */ String PROPERTY_DATA_EXTENSION_KEY = "dataExtensionKey"; /** - * Configuration property name used to specify the list of objects + * Configuration property name used to specify the list of objects. */ String PROPERTY_OBJECT_LIST = "objectList"; @@ -57,32 +57,32 @@ public interface SalesforceConstants { String PROPERTY_TABLE_NAME_FIELD = "tableNameField"; /** - * Configuration property name used to specify client id + * Configuration property name used to specify client id. */ String PROPERTY_CLIENT_ID = "clientId"; /** - * Configuration property name used to specify client secret + * Configuration property name used to specify client secret. */ String PROPERTY_CLIENT_SECRET = "clientSecret"; /** - * Configuration property name used to specify REST API endpoint + * Configuration property name used to specify REST API endpoint. */ String PROPERTY_API_ENDPOINT = "restEndpoint"; /** - * Configuration property name used to specify Auth API endpoint + * Configuration property name used to specify Auth API endpoint. */ String PROPERTY_AUTH_API_ENDPOINT = "authEndpoint"; /** - * Configuration property name used to specify SOAP endpoint + * Configuration property name used to specify SOAP endpoint. */ String PROPERTY_SOAP_API_ENDPOINT = "soapEndpoint"; /** - * Configuration property name used to specify Fail on Error flag + * Configuration property name used to specify Fail on Error flag. */ String PROPERTY_FAIL_ON_ERROR = "failOnError"; diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceObjectInfo.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceObjectInfo.java index f3cb247..f0af7af 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceObjectInfo.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceObjectInfo.java @@ -31,10 +31,24 @@ public class SalesforceObjectInfo { private final Schema schema; private final int recordCount; + /** + * Constructor for SalesforceObjectInfo for non-dataextension object. + * @param object The Salesforce Marketing Cloud object + * @param columns The list of columns + * @param recordCount The total number of records + */ public SalesforceObjectInfo(SourceObject object, List columns, int recordCount) { this(object, null, columns, recordCount); } + /** + * Constructor for SalesforceObjectInfo for dataextension object. + * + * @param object The Salesforce Marketing Cloud object as DataExtension + * @param dataExtensionKey The data extension key + * @param columns The list of columns + * @param recordCount The total number of records + */ public SalesforceObjectInfo(SourceObject object, String dataExtensionKey, List columns, int recordCount) { this.object = object; @@ -48,6 +62,12 @@ public SourceObject getObject() { return object; } + /** + * Returns the table name for the object. + * + * @return In case of Data Extension, it returns name in `dataextension_[data extension key]` format + * Otherwise, it returns object name + */ public String getTableName() { if (getObject() == SourceObject.DATA_EXTENSION) { return String.format("%s%s", DATA_EXTENSION_PREFIX, dataExtensionKey); diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SchemaBuilder.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SchemaBuilder.java index 94878e2..29f0077 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SchemaBuilder.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SchemaBuilder.java @@ -24,9 +24,16 @@ import java.util.stream.Collectors; /** - * Utility class to build schema + * Utility class to build schema. */ public class SchemaBuilder { + /** + * Constructs Schema object using input parameters. + * + * @param tableName The table name to be used in Schema object + * @param columns The list of SalesforceColumn objects that will be added as Schema.Field + * @return The instance of Schema object + */ public Schema constructSchema(String tableName, List columns) { SchemaBuilder schemaBuilder = new SchemaBuilder(); List fields = schemaBuilder.constructSchemaFields(columns); diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SourceObject.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SourceObject.java index f6e41d2..635f71a 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SourceObject.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SourceObject.java @@ -33,22 +33,22 @@ public enum SourceObject { /** - * This indicates data to be fetched from Data Extension + * This indicates data to be fetched from Data Extension. */ DATA_EXTENSION("Data Extension", "dataextension", ETDataExtensionRow.class), /** - * This indicates data to be fetched from Campaign + * This indicates data to be fetched from Campaign. */ CAMPAIGN("Campaign", "campaign", ETCampaign.class), /** - * This indicates data to be fetched from Email + * This indicates data to be fetched from Email. */ EMAIL("Email", "email", ETEmail.class), /** - * This indicates data to be fetched from Mailing List + * This indicates data to be fetched from Mailing List. */ MAILING_LIST("Mailing List", "mailinglist", ETList.class); diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SourceQueryMode.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SourceQueryMode.java index 7dfee4f..a1bbf11 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SourceQueryMode.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SourceQueryMode.java @@ -27,12 +27,12 @@ public enum SourceQueryMode { /** - * Mode will be used to fetch data for multiple objects + * Mode will be used to fetch data for multiple objects. */ MULTI_OBJECT("Multi Object"), /** - * Mode will be used to fetch data for single object + * Mode will be used to fetch data for single object. */ SINGLE_OBJECT("Single Object"); diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/Util.java b/src/main/java/io/cdap/plugin/sfmc/source/util/Util.java index ab009fe..b01af08 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/Util.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/Util.java @@ -26,10 +26,23 @@ * Utility class. */ public class Util { + /** + * Utility function to check if incoming string is empty or not. + * + * @param string The value to be checked for emptyness + * @return true if string is empty otherwise false + */ public static boolean isNullOrEmpty(String string) { return Strings.isNullOrEmpty(Strings.nullToEmpty(string).trim()); } + /** + * Utility function to split incoming string using a delimiter. + * + * @param value The value to be split + * @param delimiter The delimiter to be used for splitting + * @return The list of strings + */ public static List splitToList(String value, char delimiter) { if (isNullOrEmpty(value)) { return Collections.emptyList(); diff --git a/widgets/Salesforce-batchsource.json b/widgets/Salesforce-batchsource.json index 43b88db..2b8d6c0 100644 --- a/widgets/Salesforce-batchsource.json +++ b/widgets/Salesforce-batchsource.json @@ -219,6 +219,18 @@ "name": "tableNameField" } ] + }, + { + "name": "showSingleObjectDataExtKey", + "condition": { + "expression": "objectName == 'Data Extension'" + }, + "show": [ + { + "type": "property", + "name": "dataExtensionKey" + } + ] } ] } From 18c30d8469fbf52912f484af1471374323dbf70e Mon Sep 17 00:00:00 2001 From: amolkokane76 Date: Mon, 18 May 2020 18:19:25 +0530 Subject: [PATCH 15/19] Fixed issues related to paging and BigQuery support --- docs/Salesforce-batchsource.md | 2 +- .../plugin/sfmc/source/SalesforceClient.java | 109 ++++++++------- .../sfmc/source/SalesforceInputFormat.java | 47 ++++--- .../sfmc/source/SalesforceInputSplit.java | 24 ++-- .../source/SalesforceJobConfiguration.java | 2 +- .../sfmc/source/SalesforceRecordReader.java | 36 +++-- .../plugin/sfmc/source/SalesforceSource.java | 33 +++-- .../sfmc/source/SalesforceSourceConfig.java | 127 ++++++++---------- .../sfmc/source/util/SalesforceColumn.java | 12 +- .../sfmc/source/util/SalesforceConstants.java | 13 +- .../source/util/SalesforceObjectInfo.java | 44 ++++-- .../sfmc/source/util/SchemaBuilder.java | 6 +- .../plugin/sfmc/source/util/SourceObject.java | 13 +- .../sfmc/source/util/SourceQueryMode.java | 2 +- widgets/Salesforce-batchsource.json | 17 ++- 15 files changed, 293 insertions(+), 194 deletions(-) diff --git a/docs/Salesforce-batchsource.md b/docs/Salesforce-batchsource.md index 55ecd59..fbc5907 100644 --- a/docs/Salesforce-batchsource.md +++ b/docs/Salesforce-batchsource.md @@ -61,7 +61,7 @@ Note, this value will be ignored if the Mode is set to `Single Object`. **Table Name Field**: The name of the field that holds the object name to which the data belongs to. Must not be the name of any column for any of the objects that will be read. Defaults to `tablename`. In case of `Data Extension` -object, this field will have value in `dataextension-[Data Extension Key]` format. Note, the Table name field value +object, this field will have value in `dataextension_[Data Extension Key]` format. Note, the Table name field value will be ignored if the Mode is set to `Single Object`. **Client ID**: OAuth2 client ID associated with an installed package in the Salesforce Marketing Cloud. diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceClient.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceClient.java index fb73991..7155fd4 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceClient.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceClient.java @@ -1,5 +1,5 @@ /* - * Copyright © 2017-2019 Cask Data, Inc. + * Copyright © 2020 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -27,10 +27,9 @@ import com.exacttarget.fuelsdk.ETResponse; import com.exacttarget.fuelsdk.ETSdkException; import io.cdap.plugin.sfmc.source.util.SalesforceColumn; +import io.cdap.plugin.sfmc.source.util.SalesforceConstants; import io.cdap.plugin.sfmc.source.util.SalesforceObjectInfo; import io.cdap.plugin.sfmc.source.util.SourceObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.Arrays; import java.util.Collections; @@ -41,7 +40,6 @@ * Utility class that provides methods to connect to Salesforce instance. */ public class SalesforceClient { - private static final Logger LOG = LoggerFactory.getLogger(SalesforceClient.class); private final ETClient client; SalesforceClient(ETClient client) { @@ -51,10 +49,10 @@ public class SalesforceClient { /** * Initialize the connection with Salesforce Marketing Cloud using FuelSDK. * - * @param clientId The Salesforce Marketing Cloud Client Id - * @param clientSecret The Salesforce Marketing Cloud Client Secret - * @param authEndpoint Auth Endpoint url for Salesforce Marketing Cloud - * @param soapEndpoint SOAP Endpoint url for Salesforce Marketing Cloud + * @param clientId The Salesforce Marketing Cloud Client Id + * @param clientSecret The Salesforce Marketing Cloud Client Secret + * @param authEndpoint Auth Endpoint url for Salesforce Marketing Cloud + * @param soapEndpoint SOAP Endpoint url for Salesforce Marketing Cloud * @return The instance of SalesforceClient object * @throws ETSdkException The FuelSDKException */ @@ -78,13 +76,22 @@ public static SalesforceClient create(String clientId, String clientSecret, Stri /** * Fetch records for passed object from Salesforce Marketing Cloud. * - * @param object The SourceObject which tells what data to be fetched from Salesforce - * Marketing Cloud + * @param object The SourceObject which tells what data to be fetched from Salesforce Marketing Cloud + * @param page The page index + * @param pageSize The size of the page * @return The list of ETApiObject representing the records from requested object */ - public List fetchObjectRecords(SourceObject object) { + public List fetchObjectRecords(SourceObject object, int page, int pageSize) { try { - return fetchObjectData(client, object.getClassRef()); + Integer pageIndex = null; + Integer maxPageSize = null; + + if (object.isPagingSupported()) { + pageIndex = page; + maxPageSize = pageSize; + } + + return fetchObjectData(client, object.getClassRef(), pageIndex, maxPageSize); } catch (ETSdkException e) { return Collections.emptyList(); } @@ -93,53 +100,47 @@ public List fetchObjectRecords(SourceObject object) { /** * Fetch the schema information for passed object from Salesforce Marketing Cloud. * - * @param object The SourceObject which tells for which schema to be fetched from Salesforce - * Marketing Cloud + * @param object The SourceObject which tells for which schema to be fetched from Salesforce Marketing Cloud + * @param fetchRecordCount the flag to decide whether to fetch record count or not * @return The instance of SalesforceObjectInfo object + * @throws ETSdkException The FuelSDKException */ - public SalesforceObjectInfo fetchObjectSchema(SourceObject object) { - return new SalesforceObjectInfo(object, fetchObjectFields(object.getClassRef()), 0); + public SalesforceObjectInfo fetchObjectSchema(SourceObject object, boolean fetchRecordCount) throws ETSdkException { + Class clazz = object.getClassRef(); + Integer recordCount = SalesforceConstants.MAX_PAGE_SIZE; + + if (object.isPagingSupported() && fetchRecordCount) { + ETResponse etResponse = client.retrieve(clazz, 1, 1, new ETFilter()); + recordCount = etResponse.getTotalCount(); + } + + return new SalesforceObjectInfo(object, fetchObjectFields(clazz), recordCount); } /** * Fetch records for passed object from Salesforce Marketing Cloud. * - * @param dataExtensionKey The data extension key for which data to be fetched from Salesforce - * Marketing Cloud + * @param dataExtensionKey The data extension key for which data to be fetched from Salesforce Marketing Cloud + * @param page The page index + * @param pageSize The size of the page * @return The list of ETDataExtensionRow representing the records from requested data extension * @throws ETSdkException The FuelSDKException */ - public List fetchDataExtensionRecords(String dataExtensionKey) + public List fetchDataExtensionRecords(String dataExtensionKey, int page, int pageSize) throws ETSdkException { - return call(() -> { - ETExpression expression = buildDataExtensionExpression(dataExtensionKey); - - ETFilter filter = new ETFilter(); - filter.addProperty("key"); - filter.addProperty("name"); - filter.setExpression(expression); - - ETResponse response = client.retrieve(ETDataExtension.class, filter); - List extensionRows = response.getObjects(); - - if (extensionRows.isEmpty()) { - return Collections.emptyList(); - } - - ETDataExtension de = extensionRows.get(0); - return de.select().getObjects(); - }); + return call(() -> ETDataExtension.select(client, "key=" + dataExtensionKey, page, pageSize) + .getObjects()); } /** * Fetch the schema information for passed object from Salesforce Marketing Cloud. * - * @param dataExtensionKey The data extension key for which schema to be fetched from - * Salesforce Marketing Cloud + * @param dataExtensionKey The data extension key for which schema to be fetched from Salesforce Marketing Cloud + * @param fetchRecordCount the flag to decide whether to fetch record count or not * @return The instance of SalesforceObjectInfo object * @throws ETSdkException The FuelSDKException */ - public SalesforceObjectInfo fetchDataExtensionSchema(String dataExtensionKey) + public SalesforceObjectInfo fetchDataExtensionSchema(String dataExtensionKey, boolean fetchRecordCount) throws ETSdkException { return call(() -> { ETExpression expression = buildDataExtensionExpression(dataExtensionKey); @@ -155,10 +156,27 @@ public SalesforceObjectInfo fetchDataExtensionSchema(String dataExtensionKey) .map(o -> new SalesforceColumn(o.getName(), o.getType().name())) .collect(Collectors.toList()); - return new SalesforceObjectInfo(SourceObject.DATA_EXTENSION, dataExtensionKey, columns, 0); + Integer recordCount = fetchDataExtensionRecordCount(dataExtensionKey); + + return new SalesforceObjectInfo(SourceObject.DATA_EXTENSION, dataExtensionKey, columns, recordCount); }); } + private Integer fetchDataExtensionRecordCount(String dataExtensionKey) throws ETSdkException { + Integer totalCount = 0; + Integer pageRecordCount; + int page = 1; + + do { + pageRecordCount = ETDataExtension.select(client, "key=" + dataExtensionKey, page, + SalesforceConstants.MAX_PAGE_SIZE).getResults().size(); + totalCount += pageRecordCount; + page++; + } while (pageRecordCount >= SalesforceConstants.MAX_PAGE_SIZE); + + return totalCount; + } + private ETExpression buildDataExtensionExpression(String dataExtensionKey) { ETExpression expression = new ETExpression(); @@ -169,12 +187,11 @@ private ETExpression buildDataExtensionExpression(String dataExtensionKey) { return expression; } - private List fetchObjectData(ETClient client, Class clazz) + private List fetchObjectData(ETClient client, Class clazz, Integer page, + Integer pageSize) throws ETSdkException { - ETResponse etResponse = client.retrieve(clazz, new ETFilter()); - List rows = etResponse.getObjects(); - - return rows; + ETResponse etResponse = client.retrieve(clazz, page, pageSize, new ETFilter()); + return etResponse.getObjects(); } private List fetchObjectFields(Class clazz) { diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java index c4e2bcf..e5f823d 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java @@ -1,5 +1,5 @@ /* - * Copyright © 2017-2019 Cask Data, Inc. + * Copyright © 2020 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -17,6 +17,7 @@ package io.cdap.plugin.sfmc.source; import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.plugin.sfmc.source.util.SalesforceConstants; import io.cdap.plugin.sfmc.source.util.SalesforceObjectInfo; import io.cdap.plugin.sfmc.source.util.SourceObject; import io.cdap.plugin.sfmc.source.util.SourceQueryMode; @@ -36,8 +37,6 @@ import java.util.Collections; import java.util.List; -import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.MAX_PAGE_SIZE; - /** * Salesforce input format. */ @@ -48,17 +47,18 @@ public class SalesforceInputFormat extends InputFormat setInput(Configuration jobConfig, SourceQueryMode mode, - SalesforceSourceConfig conf) { + SalesforceSourceConfig conf, boolean fetchRecordCount) { SalesforceJobConfiguration jobConf = new SalesforceJobConfiguration(jobConfig); jobConf.setPluginConfiguration(conf); //Depending on the selected objects in the conf, get the schema for each object as SalesforceObjectInfo - List tableInfos = fetchTableInfo(mode, conf); + List tableInfos = fetchTableInfo(mode, conf, fetchRecordCount); jobConf.setTableInfos(tableInfos); @@ -69,11 +69,13 @@ public static List setInput(Configuration jobConfig, Sourc /** * Depending on conf value fetch the list of fields for each object and create schema object. * - * @param mode the query mode - * @param conf the plugin conf + * @param mode the query mode + * @param conf the plugin conf + * @param fetchRecordCount the flag to decide whether to fetch record count or not * @return Collection of SalesforceObjectInfo containing table and schema. */ - private static List fetchTableInfo(SourceQueryMode mode, SalesforceSourceConfig conf) { + private static List fetchTableInfo(SourceQueryMode mode, SalesforceSourceConfig conf, + boolean fetchRecordCount) { try { SalesforceClient client = SalesforceClient.create(conf.getClientId(), conf.getClientSecret(), conf.getAuthEndpoint(), conf.getSoapEndpoint()); @@ -83,7 +85,7 @@ private static List fetchTableInfo(SourceQueryMode mode, S //When mode = SingleObject, fetch fields for the object selected in plugin config if (mode == SourceQueryMode.SINGLE_OBJECT) { SalesforceObjectInfo tableInfo = getTableMetaData(conf.getObject(), conf.getDataExtensionKey(), client, - failOnError); + failOnError, fetchRecordCount); return (tableInfo == null) ? Collections.emptyList() : Collections.singletonList(tableInfo); } @@ -100,14 +102,14 @@ private static List fetchTableInfo(SourceQueryMode mode, S //the data extension objects. List dataExtensionKeys = Util.splitToList(conf.getDataExtensionKeys(), ','); for (String dataExtensionKey : dataExtensionKeys) { - tableInfo = getTableMetaData(object, dataExtensionKey, client, failOnError); + tableInfo = getTableMetaData(object, dataExtensionKey, client, failOnError, fetchRecordCount); if (tableInfo == null) { continue; } tableInfos.add(tableInfo); } } else { - tableInfo = getTableMetaData(object, "", client, failOnError); + tableInfo = getTableMetaData(object, "", client, failOnError, fetchRecordCount); if (tableInfo != null) { tableInfos.add(tableInfo); } @@ -129,12 +131,13 @@ private static List fetchTableInfo(SourceQueryMode mode, S * Fetch the fields for passed object. */ private static SalesforceObjectInfo getTableMetaData(SourceObject object, String dataExtensionKey, - SalesforceClient client, boolean failOnError) { + SalesforceClient client, boolean failOnError, + boolean fetchRecordCount) { try { if (object == SourceObject.DATA_EXTENSION) { - return client.fetchDataExtensionSchema(dataExtensionKey); + return client.fetchDataExtensionSchema(dataExtensionKey, fetchRecordCount); } else { - return client.fetchObjectSchema(object); + return client.fetchObjectSchema(object, fetchRecordCount); } } catch (Exception e) { if (failOnError) { @@ -157,7 +160,17 @@ public List getSplits(JobContext jobContext) throws IOException, Int for (SalesforceObjectInfo tableInfo : tableInfos) { String tableKey = tableInfo.getObject().name(); String tableName = tableInfo.getTableName(); - resultSplits.add(new SalesforceInputSplit(tableKey, tableName, 1, MAX_PAGE_SIZE)); + int totalRecords = tableInfo.getRecordCount(); + if (totalRecords <= SalesforceConstants.MAX_PAGE_SIZE) { + //add single split for table and continue + resultSplits.add(new SalesforceInputSplit(tableKey, tableName, 1, totalRecords)); + continue; + } + + int pages = (tableInfo.getRecordCount() / SalesforceConstants.MAX_PAGE_SIZE) + 1; + for (int page = 1; page <= pages; page++) { + resultSplits.add(new SalesforceInputSplit(tableKey, tableName, page, SalesforceConstants.MAX_PAGE_SIZE)); + } } LOG.debug("# of split = {}", resultSplits.size()); diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputSplit.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputSplit.java index 9d146fe..ec24f61 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputSplit.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputSplit.java @@ -1,5 +1,5 @@ /* - * Copyright © 2017 Cask Data, Inc. + * Copyright © 2020 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -30,7 +30,7 @@ public class SalesforceInputSplit extends InputSplit implements Writable { private String objectName; private String tableName; private int page; - private int length; + private int pageSize; // used by mapreduce public SalesforceInputSplit() { @@ -39,16 +39,16 @@ public SalesforceInputSplit() { /** * Constructor for SalesforceInputSplit. * - * @param objectName The object name - * @param tableName The corresponding table name - * @param page The page index - * @param length The page size + * @param objectName The object name + * @param tableName The corresponding table name + * @param page The page index + * @param pageSize The page size */ - public SalesforceInputSplit(String objectName, String tableName, int page, int length) { + public SalesforceInputSplit(String objectName, String tableName, int page, int pageSize) { this.objectName = objectName; this.tableName = tableName; this.page = page; - this.length = length; + this.pageSize = pageSize; } public String getObjectName() { @@ -63,11 +63,16 @@ public int getPage() { return page; } + public int getPageSize() { + return pageSize; + } + @Override public void write(DataOutput dataOutput) throws IOException { dataOutput.writeUTF(this.objectName); dataOutput.writeUTF(this.tableName); dataOutput.writeInt(this.page); + dataOutput.writeInt(this.pageSize); } @Override @@ -75,11 +80,12 @@ public void readFields(DataInput dataInput) throws IOException { this.objectName = dataInput.readUTF(); this.tableName = dataInput.readUTF(); this.page = dataInput.readInt(); + this.pageSize = dataInput.readInt(); } @Override public long getLength() throws IOException, InterruptedException { - return this.length; + return 0; } @Override diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceJobConfiguration.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceJobConfiguration.java index 66fcbce..27c707d 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceJobConfiguration.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceJobConfiguration.java @@ -1,5 +1,5 @@ /* - * Copyright © 2017-2019 Cask Data, Inc. + * Copyright © 2020 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java index 5e87d8d..056e51e 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java @@ -1,5 +1,5 @@ /* - * Copyright © 2017-2019 Cask Data, Inc. + * Copyright © 2020 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -21,6 +21,7 @@ import com.google.common.base.Strings; import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.sfmc.source.util.SalesforceConstants; import io.cdap.plugin.sfmc.source.util.SalesforceObjectInfo; import io.cdap.plugin.sfmc.source.util.SourceObject; import io.cdap.plugin.sfmc.source.util.SourceQueryMode; @@ -33,14 +34,11 @@ import java.io.IOException; import java.lang.reflect.Method; -import java.sql.SQLException; import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; import java.util.List; -import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.DATA_EXTENSION_PREFIX; - /** * Record reader that reads the entire contents of a Salesforce table. */ @@ -50,6 +48,7 @@ public class SalesforceRecordReader extends RecordReader tableFields; + private SalesforceObjectInfo sfObjectMetaData; private Schema schema; private SourceObject object; @@ -114,10 +113,10 @@ public StructuredRecord getCurrentValue() throws IOException { } catch (Exception e) { if (pluginConf.isFailOnError()) { - LOG.error("Error decoding row from table " + tableName, e); - throw new IOException("Error decoding row from table " + tableName, e); + LOG.error(String.format("Error decoding row from table %s", tableName), e); + throw new IOException(String.format("Error decoding row from table %s", tableName), e); } else { - LOG.warn("Failed decoding row from table " + tableName, e); + LOG.warn(String.format("Failed decoding row from table %s", tableName), e); } } return recordBuilder.build(); @@ -130,18 +129,13 @@ public float getProgress() throws IOException, InterruptedException { @Override public void close() throws IOException { - SQLException exception = null; - - if (exception != null) { - throw new IOException(exception); - } } private void fetchData() { object = SourceObject.valueOf(split.getObjectName()); tableName = split.getTableName(); if (object == SourceObject.DATA_EXTENSION) { - dataExtensionKey = tableName.replaceAll(DATA_EXTENSION_PREFIX, ""); + dataExtensionKey = tableName.replaceAll(SalesforceConstants.DATA_EXTENSION_PREFIX, ""); } tableNameField = pluginConf.getTableNameField(); @@ -152,9 +146,9 @@ private void fetchData() { //Fetch data if (object == SourceObject.DATA_EXTENSION) { - results = client.fetchDataExtensionRecords(dataExtensionKey); + results = client.fetchDataExtensionRecords(dataExtensionKey, split.getPage(), split.getPageSize()); } else { - results = client.fetchObjectRecords(object); + results = client.fetchObjectRecords(object, split.getPage(), split.getPageSize()); } LOG.debug("size={}", results.size()); @@ -178,15 +172,14 @@ private void fetchSchema(SalesforceClient client) { List schemaFields; try { - SalesforceObjectInfo metaData = null; if (object == SourceObject.DATA_EXTENSION) { - metaData = client.fetchDataExtensionSchema(dataExtensionKey); + sfObjectMetaData = client.fetchDataExtensionSchema(dataExtensionKey, false); } else { - metaData = client.fetchObjectSchema(object); + sfObjectMetaData = client.fetchObjectSchema(object, false); } //Build schema - tableFields = metaData.getSchema().getFields(); + tableFields = sfObjectMetaData.getSchema().getFields(); schemaFields = new ArrayList<>(tableFields); if (pluginConf.getQueryMode() == SourceQueryMode.MULTI_OBJECT) { @@ -196,7 +189,7 @@ private void fetchSchema(SalesforceClient client) { schemaFields = Collections.emptyList(); } - schema = Schema.recordOf(tableName, schemaFields); + schema = Schema.recordOf(tableName.replaceAll("-", "_"), schemaFields); } /** @@ -207,7 +200,8 @@ private void convertRecord(StructuredRecord.Builder recordBuilder, ETApiObject r String fieldName = field.getName(); Object rawFieldValue = null; if (row instanceof ETDataExtensionRow) { - rawFieldValue = ((ETDataExtensionRow) row).getColumn(fieldName); + String apiFieldName = sfObjectMetaData.lookupFieldsMap(fieldName); + rawFieldValue = ((ETDataExtensionRow) row).getColumn(apiFieldName); } else { rawFieldValue = getFieldValue(row, fieldName); } diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java index 230523a..d4cded5 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java @@ -1,5 +1,5 @@ /* - * Copyright © 2017-2019 Cask Data, Inc. + * Copyright © 2020 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -33,7 +33,9 @@ import io.cdap.cdap.etl.api.batch.BatchSourceContext; import io.cdap.plugin.common.LineageRecorder; import io.cdap.plugin.common.SourceInputFormatProvider; +import io.cdap.plugin.sfmc.source.util.SalesforceConstants; import io.cdap.plugin.sfmc.source.util.SalesforceObjectInfo; +import io.cdap.plugin.sfmc.source.util.SourceObject; import io.cdap.plugin.sfmc.source.util.SourceQueryMode; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.NullWritable; @@ -48,14 +50,11 @@ import java.util.Objects; import java.util.stream.Collectors; -import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PLUGIN_NAME; -import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.TABLE_PREFIX; - /** * A {@link BatchSource} that reads data from multiple objects in Salesforce. */ @Plugin(type = BatchSource.PLUGIN_TYPE) -@Name(PLUGIN_NAME) +@Name(SalesforceConstants.PLUGIN_NAME) @Description("Reads from multiple objects in Salesforce. " + "Outputs one record for each row in each object, with the object name as a record field. " + "Also sets a pipeline argument for each object read, which contains the object schema.") @@ -82,6 +81,9 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) { // getSchema method call collector.getOrThrowException(); + //Get Schema + stageConfigurer.setOutputSchema(getSchema(conf.getQueryMode())); + if (conf.isFailOnError()) { if (pipelineConfigurer.getEngine() == Engine.SPARK) { pipelineConfigurer.setPipelineProperties(Collections.singletonMap("spark.task.maxFailures", "1")); @@ -103,10 +105,11 @@ public void prepareRun(BatchSourceContext context) throws Exception { SourceQueryMode mode = conf.getQueryMode(collector); Configuration hConf = new Configuration(); - Collection tables = SalesforceInputFormat.setInput(hConf, mode, conf); + Collection tables = SalesforceInputFormat.setInput(hConf, mode, conf, true); SettableArguments arguments = context.getArguments(); for (SalesforceObjectInfo tableInfo : tables) { - arguments.set(TABLE_PREFIX + tableInfo.getTableName(), tableInfo.getSchema().toString()); + arguments.set(SalesforceConstants.TABLE_PREFIX + tableInfo.getFormattedTableName(), + tableInfo.getSchema().toString()); recordLineage(context, tableInfo); } @@ -119,8 +122,22 @@ public void transform(KeyValue input, Emitter tables = SalesforceInputFormat.setInput(hConf, mode, conf, false); + if (tables != null && !tables.isEmpty()) { + schema = tables.iterator().next().getSchema(); + } + } + + return schema; + } + private void recordLineage(BatchSourceContext context, SalesforceObjectInfo tableInfo) { - String tableName = tableInfo.getTableName(); + String tableName = tableInfo.getFormattedTableName(); String outputName = String.format("%s-%s", conf.getReferenceName(), tableName); Schema schema = tableInfo.getSchema(); LineageRecorder lineageRecorder = new LineageRecorder(context, outputName); diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java index 5ed773b..5e58346 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java @@ -1,5 +1,5 @@ /* - * Copyright © 2017-2019 Cask Data, Inc. + * Copyright © 2020 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -25,6 +25,7 @@ import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.plugin.common.Constants; import io.cdap.plugin.common.IdUtils; +import io.cdap.plugin.sfmc.source.util.SalesforceConstants; import io.cdap.plugin.sfmc.source.util.SourceObject; import io.cdap.plugin.sfmc.source.util.SourceQueryMode; import io.cdap.plugin.sfmc.source.util.Util; @@ -36,19 +37,6 @@ import java.util.Optional; import javax.annotation.Nullable; -import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_API_ENDPOINT; -import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_AUTH_API_ENDPOINT; -import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_CLIENT_ID; -import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_CLIENT_SECRET; -import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_DATA_EXTENSION_KEY; -import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_DATA_EXTENSION_KEY_LIST; -import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_FAIL_ON_ERROR; -import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_OBJECT_LIST; -import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_OBJECT_NAME; -import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_QUERY_MODE; -import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_SOAP_API_ENDPOINT; -import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_TABLE_NAME_FIELD; - /** * Configuration for the {@link SalesforceSource}. */ @@ -59,14 +47,14 @@ public class SalesforceSourceConfig extends PluginConfig { @Description("This will be used to uniquely identify this source for lineage, annotating metadata, etc.") private String referenceName; - @Name(PROPERTY_QUERY_MODE) + @Name(SalesforceConstants.PROPERTY_QUERY_MODE) @Macro @Description("Mode of data retrieval. The mode can be one of two values: " + "`Multi Object` - will allow user to fetch data for multiple data extensions, " + "`Single Object` - will allow user to fetch data for single data extension.") private String queryMode; - @Name(PROPERTY_OBJECT_NAME) + @Name(SalesforceConstants.PROPERTY_OBJECT_NAME) @Macro @Nullable @Description("Specify the object for which data to be fetched. This can be one of following values: " + @@ -77,7 +65,7 @@ public class SalesforceSourceConfig extends PluginConfig { "Note, this value will be ignored if the Mode is set to `Multi Object`.") private String objectName; - @Name(PROPERTY_DATA_EXTENSION_KEY) + @Name(SalesforceConstants.PROPERTY_DATA_EXTENSION_KEY) @Macro @Nullable @Description("Specify the data extension key from which data to be fetched. Note, this value will be ignored in " + @@ -85,7 +73,7 @@ public class SalesforceSourceConfig extends PluginConfig { "`Data Extension`.") private String dataExtensionKey; - @Name(PROPERTY_OBJECT_LIST) + @Name(SalesforceConstants.PROPERTY_OBJECT_LIST) @Macro @Nullable @Description("Specify the comma-separated list of objects for which data to be fetched; for example: " + @@ -97,7 +85,7 @@ public class SalesforceSourceConfig extends PluginConfig { "Note, this value will be ignored if the Mode is set to `Single Object`.") private String objectList; - @Name(PROPERTY_DATA_EXTENSION_KEY_LIST) + @Name(SalesforceConstants.PROPERTY_DATA_EXTENSION_KEY_LIST) @Macro @Nullable @Description("Specify the data extension keys from which data to be fetched; for example: 'Key1,Key2'. " + @@ -105,38 +93,38 @@ public class SalesforceSourceConfig extends PluginConfig { "2. If the selected object list does not contain `Data Extension` as one of the objects.") private String dataExtensionKeys; - @Name(PROPERTY_TABLE_NAME_FIELD) + @Name(SalesforceConstants.PROPERTY_TABLE_NAME_FIELD) @Macro @Nullable @Description("The name of the field that holds the object name to which the data belongs to. Must not be the name " + "of any column for any of the objects that will be read. Defaults to `tablename`. In case of `Data Extension` " + - "object, this field will have value in `dataextension-[Data Extension Key]` format. Note, the Table name field " + + "object, this field will have value in `dataextension_[Data Extension Key]` format. Note, the Table name field " + "value will be ignored if the Mode is set to `Single Object`.") private String tableNameField; - @Name(PROPERTY_CLIENT_ID) + @Name(SalesforceConstants.PROPERTY_CLIENT_ID) @Macro @Description("OAuth2 client ID associated with an installed package in the Salesforce Marketing Cloud.") private String clientId; - @Name(PROPERTY_CLIENT_SECRET) + @Name(SalesforceConstants.PROPERTY_CLIENT_SECRET) @Macro @Description("OAuth2 client secret associated with an installed package in the Salesforce Marketing Cloud.") private String clientSecret; - @Name(PROPERTY_API_ENDPOINT) + @Name(SalesforceConstants.PROPERTY_API_ENDPOINT) @Macro @Description("The REST API Base URL associated for the Server-to-Server API integration. " + "For example, https://instance.rest.marketingcloudapis.com/") private String restEndpoint; - @Name(PROPERTY_AUTH_API_ENDPOINT) + @Name(SalesforceConstants.PROPERTY_AUTH_API_ENDPOINT) @Macro @Description("Authentication Base URL associated for the Server-to-Server API integration. " + "For example, https://instance.auth.marketingcloudapis.com/") private String authEndpoint; - @Name(PROPERTY_SOAP_API_ENDPOINT) + @Name(SalesforceConstants.PROPERTY_SOAP_API_ENDPOINT) @Macro @Description("The SOAP Endpoint URL associated for the Server-to-Server API integration. " + "For example, https://instance.soap.marketingcloudapis.com/Service.asmx") @@ -144,26 +132,26 @@ public class SalesforceSourceConfig extends PluginConfig { @Macro @Nullable - @Name(PROPERTY_FAIL_ON_ERROR) + @Name(SalesforceConstants.PROPERTY_FAIL_ON_ERROR) @Description("Whether to fail the pipeline if it fails while reading.") private Boolean failOnError; /** * Constructor for SalesforceSourceConfig object. * - * @param referenceName The reference name - * @param queryMode The query mode - * @param objectName The object name to be fetched from Salesforce Marketing Cloud - * @param dataExtensionKey The data extension key to be fetched from Salesforce Marketing Cloud - * @param objectList The list of objects to be fetched from Salesforce Marketing Cloud + * @param referenceName The reference name + * @param queryMode The query mode + * @param objectName The object name to be fetched from Salesforce Marketing Cloud + * @param dataExtensionKey The data extension key to be fetched from Salesforce Marketing Cloud + * @param objectList The list of objects to be fetched from Salesforce Marketing Cloud * @param dataExtensionKeys The list of data extension keys to be fetched from Salesforce Marketing Cloud - * @param tableNameField The field name to hold the table name value - * @param clientId The Salesforce Marketing Cloud Client Id - * @param clientSecret The Salesforce Marketing Cloud Client Secret - * @param restEndpoint The REST API endpoint for Salesforce Marketing Cloud - * @param authEndpoint The AUTH API endpoint for Salesforce Marketing Cloud - * @param soapEndpoint The SOAP API endpoint for Salesforce Marketing Cloud - * @param failOnError A flag to decide what is to be done to the pipeline execution in case of errors + * @param tableNameField The field name to hold the table name value + * @param clientId The Salesforce Marketing Cloud Client Id + * @param clientSecret The Salesforce Marketing Cloud Client Secret + * @param restEndpoint The REST API endpoint for Salesforce Marketing Cloud + * @param authEndpoint The AUTH API endpoint for Salesforce Marketing Cloud + * @param soapEndpoint The SOAP API endpoint for Salesforce Marketing Cloud + * @param failOnError A flag to decide what is to be done to the pipeline execution in case of errors */ public SalesforceSourceConfig(String referenceName, String queryMode, @Nullable String objectName, @Nullable String dataExtensionKey, @Nullable String objectList, @@ -192,7 +180,7 @@ public String getReferenceName() { /** * Returns the query mode chosen. * - * @param collector The failure collector to collect the errors + * @param collector The failure collector to collect the errors * @return An instance of SourceQueryMode */ public SourceQueryMode getQueryMode(FailureCollector collector) { @@ -203,7 +191,7 @@ public SourceQueryMode getQueryMode(FailureCollector collector) { collector.addFailure("Unsupported query mode value: " + queryMode, String.format("Supported modes are: %s", SourceQueryMode.getSupportedModes())) - .withConfigProperty(PROPERTY_QUERY_MODE); + .withConfigProperty(SalesforceConstants.PROPERTY_QUERY_MODE); collector.getOrThrowException(); return null; } @@ -222,7 +210,7 @@ public SourceQueryMode getQueryMode() { /** * Returns selected object. * - * @param collector The failure collector to collect the errors + * @param collector The failure collector to collect the errors * @return An instance of SourceObject */ public SourceObject getObject(FailureCollector collector) { @@ -233,7 +221,7 @@ public SourceObject getObject(FailureCollector collector) { collector.addFailure("Unsupported object value: " + objectName, String.format("Supported objects are: %s", SourceObject.getSupportedObjects())) - .withConfigProperty(PROPERTY_OBJECT_NAME); + .withConfigProperty(SalesforceConstants.PROPERTY_OBJECT_NAME); collector.getOrThrowException(); return null; } @@ -251,7 +239,7 @@ public String getDataExtensionKey() { /** * Returns list of selected objects. * - * @param collector The failure collector to collect the errors + * @param collector The failure collector to collect the errors * @return The list of SourceObject */ public List getObjectList(FailureCollector collector) { @@ -263,7 +251,7 @@ public List getObjectList(FailureCollector collector) { if (sourceObject == null) { collector.addFailure("Unsupported object value: " + object, String.format("Supported objects are: %s", SourceObject.getSupportedObjects())) - .withConfigProperty(PROPERTY_OBJECT_LIST); + .withConfigProperty(SalesforceConstants.PROPERTY_OBJECT_LIST); break; } @@ -325,7 +313,6 @@ public String getSoapEndpoint() { return soapEndpoint; } - @Nullable public Boolean isFailOnError() { return failOnError == null ? false : failOnError; } @@ -354,27 +341,27 @@ private void validateCredentials(FailureCollector collector) { if (Util.isNullOrEmpty(clientId)) { collector.addFailure("Client ID must be specified.", null) - .withConfigProperty(PROPERTY_CLIENT_ID); + .withConfigProperty(SalesforceConstants.PROPERTY_CLIENT_ID); } if (Util.isNullOrEmpty(clientSecret)) { collector.addFailure("Client Secret must be specified.", null) - .withConfigProperty(PROPERTY_CLIENT_SECRET); + .withConfigProperty(SalesforceConstants.PROPERTY_CLIENT_SECRET); } if (Util.isNullOrEmpty(restEndpoint)) { collector.addFailure(" REST Endpoint must be specified.", null) - .withConfigProperty(PROPERTY_API_ENDPOINT); + .withConfigProperty(SalesforceConstants.PROPERTY_API_ENDPOINT); } if (Util.isNullOrEmpty(authEndpoint)) { collector.addFailure("Auth Endpoint must be specified.", null) - .withConfigProperty(PROPERTY_AUTH_API_ENDPOINT); + .withConfigProperty(SalesforceConstants.PROPERTY_AUTH_API_ENDPOINT); } if (Util.isNullOrEmpty(soapEndpoint)) { collector.addFailure("Soap Endpoint must be specified.", null) - .withConfigProperty(PROPERTY_SOAP_API_ENDPOINT); + .withConfigProperty(SalesforceConstants.PROPERTY_SOAP_API_ENDPOINT); } collector.getOrThrowException(); @@ -389,18 +376,18 @@ void validateSalesforceConnection(FailureCollector collector) { collector.addFailure("Unable to connect to Salesforce Instance.", "Ensure properties like Client ID, Client Secret, API Endpoint, Soap Endpoint, Auth Endpoint " + "are correct.") - .withConfigProperty(PROPERTY_CLIENT_ID) - .withConfigProperty(PROPERTY_CLIENT_SECRET) - .withConfigProperty(PROPERTY_API_ENDPOINT) - .withConfigProperty(PROPERTY_AUTH_API_ENDPOINT) - .withConfigProperty(PROPERTY_SOAP_API_ENDPOINT) + .withConfigProperty(SalesforceConstants.PROPERTY_CLIENT_ID) + .withConfigProperty(SalesforceConstants.PROPERTY_CLIENT_SECRET) + .withConfigProperty(SalesforceConstants.PROPERTY_API_ENDPOINT) + .withConfigProperty(SalesforceConstants.PROPERTY_AUTH_API_ENDPOINT) + .withConfigProperty(SalesforceConstants.PROPERTY_SOAP_API_ENDPOINT) .withStacktrace(e.getStackTrace()); } } private void validateQueryMode(FailureCollector collector) { //according to query mode check if either object name / object list exists or not - if (containsMacro(PROPERTY_QUERY_MODE)) { + if (containsMacro(SalesforceConstants.PROPERTY_QUERY_MODE)) { return; } @@ -414,8 +401,9 @@ private void validateQueryMode(FailureCollector collector) { } private void validateMultiObjectQueryMode(FailureCollector collector) { - if (containsMacro(PROPERTY_OBJECT_LIST) || containsMacro(PROPERTY_DATA_EXTENSION_KEY_LIST) - || containsMacro(PROPERTY_TABLE_NAME_FIELD)) { + if (containsMacro(SalesforceConstants.PROPERTY_OBJECT_LIST) + || containsMacro(SalesforceConstants.PROPERTY_DATA_EXTENSION_KEY_LIST) + || containsMacro(SalesforceConstants.PROPERTY_TABLE_NAME_FIELD)) { return; } @@ -424,25 +412,26 @@ private void validateMultiObjectQueryMode(FailureCollector collector) { if (objects.isEmpty()) { collector.addFailure("At least 1 Object must be specified.", null) - .withConfigProperty(PROPERTY_OBJECT_LIST); + .withConfigProperty(SalesforceConstants.PROPERTY_OBJECT_LIST); } if (objects.contains(SourceObject.DATA_EXTENSION)) { List dataExtensionKeyList = Util.splitToList(getDataExtensionKeys(), ','); if (dataExtensionKeyList.isEmpty()) { collector.addFailure("At least 1 Data Extension Key must be specified.", null) - .withConfigProperty(PROPERTY_DATA_EXTENSION_KEY_LIST); + .withConfigProperty(SalesforceConstants.PROPERTY_DATA_EXTENSION_KEY_LIST); } } if (Util.isNullOrEmpty(tableNameField)) { collector.addFailure("Table name field must be specified.", null) - .withConfigProperty(PROPERTY_TABLE_NAME_FIELD); + .withConfigProperty(SalesforceConstants.PROPERTY_TABLE_NAME_FIELD); } } private void validateSingleObjectQueryMode(FailureCollector collector) { - if (containsMacro(PROPERTY_OBJECT_NAME) || containsMacro(PROPERTY_DATA_EXTENSION_KEY)) { + if (containsMacro(SalesforceConstants.PROPERTY_OBJECT_NAME) + || containsMacro(SalesforceConstants.PROPERTY_DATA_EXTENSION_KEY)) { return; } @@ -450,7 +439,7 @@ private void validateSingleObjectQueryMode(FailureCollector collector) { if (object == SourceObject.DATA_EXTENSION && Util.isNullOrEmpty(dataExtensionKey)) { collector.addFailure("Data Extension Key must be specified.", null) - .withConfigProperty(PROPERTY_DATA_EXTENSION_KEY); + .withConfigProperty(SalesforceConstants.PROPERTY_DATA_EXTENSION_KEY); } } @@ -458,10 +447,10 @@ private void validateSingleObjectQueryMode(FailureCollector collector) { * Returns true if Salesforce can be connected to. */ public boolean shouldConnect() { - return !containsMacro(PROPERTY_CLIENT_ID) && - !containsMacro(PROPERTY_CLIENT_SECRET) && - !containsMacro(PROPERTY_API_ENDPOINT) && - !containsMacro(PROPERTY_AUTH_API_ENDPOINT) && - !containsMacro(PROPERTY_SOAP_API_ENDPOINT); + return !containsMacro(SalesforceConstants.PROPERTY_CLIENT_ID) && + !containsMacro(SalesforceConstants.PROPERTY_CLIENT_SECRET) && + !containsMacro(SalesforceConstants.PROPERTY_API_ENDPOINT) && + !containsMacro(SalesforceConstants.PROPERTY_AUTH_API_ENDPOINT) && + !containsMacro(SalesforceConstants.PROPERTY_SOAP_API_ENDPOINT); } } diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceColumn.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceColumn.java index 7c3aa67..d4419bf 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceColumn.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceColumn.java @@ -1,5 +1,5 @@ /* - * Copyright © 2019 Cask Data, Inc. + * Copyright © 2020 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -33,7 +33,7 @@ public SalesforceColumn() { * Constructor for SalesforceColumn. * * @param fieldName The column name - * @param typeName The data type name + * @param typeName The data type name */ public SalesforceColumn(String fieldName, String typeName) { this.fieldName = fieldName; @@ -45,6 +45,14 @@ public String getFieldName() { return fieldName; } + /** + * Replaces all space characters in column name with double underscores + * @return field name with all characters replaced. + */ + public String getFormattedFieldName() { + return fieldName.replaceAll(" ", "__"); + } + public String getTypeName() { return typeName; } diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java index c236956..2125f7e 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java @@ -1,5 +1,5 @@ /* - * Copyright © 2019 Cask Data, Inc. + * Copyright © 2020 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -86,9 +86,18 @@ public interface SalesforceConstants { */ String PROPERTY_FAIL_ON_ERROR = "failOnError"; + /** + * Table prefix to be used in case of Multi-Object support. + */ String TABLE_PREFIX = "multisink."; - String DATA_EXTENSION_PREFIX = "dataextension-"; + /** + * Table name prefix to be used for data extensions. + */ + String DATA_EXTENSION_PREFIX = "dataextension_"; + /** + * The max limit for the page size. + */ int MAX_PAGE_SIZE = 2500; } diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceObjectInfo.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceObjectInfo.java index f0af7af..a309153 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceObjectInfo.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceObjectInfo.java @@ -1,5 +1,5 @@ /* - * Copyright © 2017-2019 Cask Data, Inc. + * Copyright © 2020 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -19,8 +19,8 @@ import io.cdap.cdap.api.data.schema.Schema; import java.util.List; - -import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.DATA_EXTENSION_PREFIX; +import java.util.Map; +import java.util.stream.Collectors; /** * Information about a Salesforce table. @@ -29,12 +29,13 @@ public class SalesforceObjectInfo { private final SourceObject object; private final String dataExtensionKey; private final Schema schema; + private final Map mapSchemaFieldToSFMCField; private final int recordCount; /** * Constructor for SalesforceObjectInfo for non-dataextension object. - * @param object The Salesforce Marketing Cloud object - * @param columns The list of columns + * @param object The Salesforce Marketing Cloud object + * @param columns The list of columns * @param recordCount The total number of records */ public SalesforceObjectInfo(SourceObject object, List columns, int recordCount) { @@ -44,17 +45,19 @@ public SalesforceObjectInfo(SourceObject object, List columns, /** * Constructor for SalesforceObjectInfo for dataextension object. * - * @param object The Salesforce Marketing Cloud object as DataExtension - * @param dataExtensionKey The data extension key - * @param columns The list of columns - * @param recordCount The total number of records + * @param object The Salesforce Marketing Cloud object as DataExtension + * @param dataExtensionKey The data extension key + * @param columns The list of columns + * @param recordCount The total number of records */ public SalesforceObjectInfo(SourceObject object, String dataExtensionKey, List columns, int recordCount) { this.object = object; this.dataExtensionKey = dataExtensionKey; SchemaBuilder schemaBuilder = new SchemaBuilder(); - this.schema = schemaBuilder.constructSchema(getTableName(), columns); + this.schema = schemaBuilder.constructSchema(getFormattedTableName(), columns); + this.mapSchemaFieldToSFMCField = columns.stream() + .collect(Collectors.toMap(SalesforceColumn::getFormattedFieldName, SalesforceColumn::getFieldName)); this.recordCount = recordCount; } @@ -65,17 +68,25 @@ public SourceObject getObject() { /** * Returns the table name for the object. * - * @return In case of Data Extension, it returns name in `dataextension_[data extension key]` format + * @return In case of Data Extension, it returns name in `dataextension-[data extension key]` format * Otherwise, it returns object name */ public String getTableName() { if (getObject() == SourceObject.DATA_EXTENSION) { - return String.format("%s%s", DATA_EXTENSION_PREFIX, dataExtensionKey); + return String.format("%s%s", SalesforceConstants.DATA_EXTENSION_PREFIX, dataExtensionKey); } else { return getObject().getTableName(); } } + /** + * Replaces all hyphen (-) characters in column name with single underscore + * @return table name with all characters replaced. + */ + public String getFormattedTableName() { + return getTableName().replaceAll("-", "_"); + } + public Schema getSchema() { return schema; } @@ -83,4 +94,13 @@ public Schema getSchema() { public int getRecordCount() { return recordCount; } + + public String lookupFieldsMap(String schemaFieldName) { + if (this.mapSchemaFieldToSFMCField == null) { + return schemaFieldName; + } + + String sfmcFieldName = this.mapSchemaFieldToSFMCField.get(schemaFieldName); + return sfmcFieldName == null ? schemaFieldName : sfmcFieldName; + } } diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SchemaBuilder.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SchemaBuilder.java index 29f0077..d161a45 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SchemaBuilder.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SchemaBuilder.java @@ -1,5 +1,5 @@ /* - * Copyright © 2017-2019 Cask Data, Inc. + * Copyright © 2020 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -31,7 +31,7 @@ public class SchemaBuilder { * Constructs Schema object using input parameters. * * @param tableName The table name to be used in Schema object - * @param columns The list of SalesforceColumn objects that will be added as Schema.Field + * @param columns The list of SalesforceColumn objects that will be added as Schema.Field * @return The instance of Schema object */ public Schema constructSchema(String tableName, List columns) { @@ -49,7 +49,7 @@ private List constructSchemaFields(List columns) } private Schema.Field transformToField(SalesforceColumn column) { - String name = column.getFieldName(); + String name = column.getFormattedFieldName(); if (Strings.isNullOrEmpty(name)) { return null; } diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SourceObject.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SourceObject.java index 635f71a..30db38e 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SourceObject.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SourceObject.java @@ -1,5 +1,5 @@ /* - * Copyright © 2019 Cask Data, Inc. + * Copyright © 2020 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -55,11 +55,18 @@ public enum SourceObject { private final String value; private final String tableName; private final Class classRef; + private final boolean pagingSupported; SourceObject(String value, String tableName, Class classRef) { this.value = value; this.tableName = tableName; this.classRef = classRef; + if (tableName.equals("dataextension")) { + this.pagingSupported = true; + } else { + this.pagingSupported = getClassRef().getSuperclass() != null + && !getClassRef().getSuperclass().getSimpleName().equalsIgnoreCase("ETSoapObject"); + } } /** @@ -90,4 +97,8 @@ public String getTableName() { public Class getClassRef() { return classRef; } + + public boolean isPagingSupported() { + return pagingSupported; + } } diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SourceQueryMode.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SourceQueryMode.java index a1bbf11..512e804 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SourceQueryMode.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SourceQueryMode.java @@ -1,5 +1,5 @@ /* - * Copyright © 2019 Cask Data, Inc. + * Copyright © 2020 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of diff --git a/widgets/Salesforce-batchsource.json b/widgets/Salesforce-batchsource.json index 2b8d6c0..12501b7 100644 --- a/widgets/Salesforce-batchsource.json +++ b/widgets/Salesforce-batchsource.json @@ -52,6 +52,16 @@ "name": "dataExtensionKey", "widget-attributes": { "placeholder": "External key of the Data Extension from which data to be read." + }, + "plugin-function": { + "label": "Get Schema", + "widget": "outputSchema", + "output-property": "schema", + "omit-properties": [ + { + "name": "schema" + } + ] } } ] @@ -175,7 +185,12 @@ ] } ], - "outputs": [ ], + "outputs": [ + { + "name": "schema", + "widget-type": "schema" + } + ], "jump-config": { "datasets": [ { From 406eee4e7ab15e1c7746b902547c6f7cc48560c0 Mon Sep 17 00:00:00 2001 From: Shivakumar Gokaram Date: Mon, 15 Feb 2021 19:07:21 -0500 Subject: [PATCH 16/19] Fixes for pagination , supporting additional objects with filter criteria --- README.md | 2 +- ...ource.md => MarketingCloud-batchsource.md} | 2 + ...rce.png => MarketingCloud-batchsource.png} | Bin pom.xml | 7 +- .../java/com/custom/fuelsdk/ETClickEvent.java | 189 +++++++++ .../com/custom/fuelsdk/ETNotSentEvent.java | 155 ++++++++ .../fuelsdk/PaginationETSoapObject.java | 366 ++++++++++++++++++ .../custom/fuelsdk/internal/ClickEvent.java | 133 +++++++ .../sfmc/source/MarketingCloudClient.java | 241 ++++++++++++ ...at.java => MarketingCloudInputFormat.java} | 78 ++-- ...lit.java => MarketingCloudInputSplit.java} | 25 +- ...va => MarketingCloudJobConfiguration.java} | 18 +- ...r.java => MarketingCloudRecordReader.java} | 35 +- ...eSource.java => MarketingCloudSource.java} | 30 +- ...g.java => MarketingCloudSourceConfig.java} | 185 +++++---- .../plugin/sfmc/source/SalesforceClient.java | 225 ----------- ...eColumn.java => MarketingCloudColumn.java} | 10 +- ...ants.java => MarketingCloudConstants.java} | 9 +- ...nfo.java => MarketingCloudObjectInfo.java} | 28 +- .../sfmc/source/util/SchemaBuilder.java | 8 +- .../plugin/sfmc/source/util/SourceObject.java | 68 +++- .../sfmc/source/util/SourceQueryMode.java | 2 +- .../source/SalesforceSourceConfigHelper.java | 15 +- .../source/SalesforceSourceConfigTest.java | 48 +-- ...e.json => MarketingCloud-batchsource.json} | 61 ++- 25 files changed, 1440 insertions(+), 500 deletions(-) rename docs/{Salesforce-batchsource.md => MarketingCloud-batchsource.md} (97%) rename icons/{Salesforce-batchsource.png => MarketingCloud-batchsource.png} (100%) create mode 100644 src/main/java/com/custom/fuelsdk/ETClickEvent.java create mode 100644 src/main/java/com/custom/fuelsdk/ETNotSentEvent.java create mode 100644 src/main/java/com/custom/fuelsdk/PaginationETSoapObject.java create mode 100644 src/main/java/com/custom/fuelsdk/internal/ClickEvent.java create mode 100644 src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudClient.java rename src/main/java/io/cdap/plugin/sfmc/source/{SalesforceInputFormat.java => MarketingCloudInputFormat.java} (58%) rename src/main/java/io/cdap/plugin/sfmc/source/{SalesforceInputSplit.java => MarketingCloudInputSplit.java} (72%) rename src/main/java/io/cdap/plugin/sfmc/source/{SalesforceJobConfiguration.java => MarketingCloudJobConfiguration.java} (72%) rename src/main/java/io/cdap/plugin/sfmc/source/{SalesforceRecordReader.java => MarketingCloudRecordReader.java} (87%) rename src/main/java/io/cdap/plugin/sfmc/source/{SalesforceSource.java => MarketingCloudSource.java} (81%) rename src/main/java/io/cdap/plugin/sfmc/source/{SalesforceSourceConfig.java => MarketingCloudSourceConfig.java} (64%) delete mode 100644 src/main/java/io/cdap/plugin/sfmc/source/SalesforceClient.java rename src/main/java/io/cdap/plugin/sfmc/source/util/{SalesforceColumn.java => MarketingCloudColumn.java} (86%) rename src/main/java/io/cdap/plugin/sfmc/source/util/{SalesforceConstants.java => MarketingCloudConstants.java} (92%) rename src/main/java/io/cdap/plugin/sfmc/source/util/{SalesforceObjectInfo.java => MarketingCloudObjectInfo.java} (73%) rename widgets/{Salesforce-batchsource.json => MarketingCloud-batchsource.json} (86%) diff --git a/README.md b/README.md index 573c64d..6f45f65 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Contains CDAP pipeline plugins for Salesforce Marketing Cloud. Following plugins are available in this repository. -* [Salesforce Marketing Cloud Source](docs/Salesforce-batchsource.md) +* [Salesforce Marketing Cloud Source](docs/MarketingCloud-batchsource.md) * [Salesforce Marketing Cloud Data Extension Sink](docs/SalesforceDataExtension-batchsink.md) # Contact diff --git a/docs/Salesforce-batchsource.md b/docs/MarketingCloud-batchsource.md similarity index 97% rename from docs/Salesforce-batchsource.md rename to docs/MarketingCloud-batchsource.md index fbc5907..a382ea8 100644 --- a/docs/Salesforce-batchsource.md +++ b/docs/MarketingCloud-batchsource.md @@ -64,6 +64,8 @@ name of any column for any of the objects that will be read. Defaults to `tablen object, this field will have value in `dataextension_[Data Extension Key]` format. Note, the Table name field value will be ignored if the Mode is set to `Single Object`. +**Filter**: The filter selection criteria. eg: eventDate > '2010-01-01' and subscriberKey = 1001 + **Client ID**: OAuth2 client ID associated with an installed package in the Salesforce Marketing Cloud. **Client Secret**: TOAuth2 client secret associated with an installed package in the Salesforce Marketing Cloud. diff --git a/icons/Salesforce-batchsource.png b/icons/MarketingCloud-batchsource.png similarity index 100% rename from icons/Salesforce-batchsource.png rename to icons/MarketingCloud-batchsource.png diff --git a/pom.xml b/pom.xml index c8334ab..d7b5136 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ Salesforce Marketing plugins io.cdap.plugin salesforce-marketing-plugins - 1.2.0-SNAPSHOT + 1.2.2-SNAPSHOT jar @@ -202,6 +202,11 @@ commons-lang 2.6 + + commons-collections + commons-collections + 3.2.1 + org.slf4j jcl-over-slf4j diff --git a/src/main/java/com/custom/fuelsdk/ETClickEvent.java b/src/main/java/com/custom/fuelsdk/ETClickEvent.java new file mode 100644 index 0000000..748e4aa --- /dev/null +++ b/src/main/java/com/custom/fuelsdk/ETClickEvent.java @@ -0,0 +1,189 @@ +/* + * Copyright © 2020 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.custom.fuelsdk; + +import com.custom.fuelsdk.internal.ClickEvent; +import com.exacttarget.fuelsdk.ETSoapObject; +import com.exacttarget.fuelsdk.annotations.ExternalName; +import com.exacttarget.fuelsdk.annotations.InternalName; +import com.exacttarget.fuelsdk.annotations.SoapObject; +import com.exacttarget.fuelsdk.internal.EventType; + +import java.util.Date; + +/** + * An ETClickEvent object represents information pertaining to the + * click events of an email message in the Salesforce Marketing Cloud. + */ +@SoapObject(internalType = ClickEvent.class) +public class ETClickEvent extends ETSoapObject { + + @InternalName("objectID") + private String id; + + @ExternalName("sendID") + private Integer sendID; + + @ExternalName("subscriberKey") + private String subscriberKey; + + @ExternalName("eventDate") + private Date eventDate; + + @ExternalName("eventType") + private EventType eventType; + + @ExternalName("triggeredSendDefinitionObjectID") + private String triggeredSendDefinitionObjectID; + + @ExternalName("batchID") + private Integer batchID; + + @ExternalName("urlid") + private Integer urlid; + + @ExternalName("url") + private String url; + + /** + * @return the id + */ + @Override + public String getId() { + return id; + } + + /** + * @param id the id to set + */ + @Override + public void setId(String id) { + this.id = id; + } + + /** + * @return the sendID + */ + public Integer getSendID() { + return sendID; + } + + /** + * @param sendID the sendID to set + */ + public void setSendID(Integer sendID) { + this.sendID = sendID; + } + + /** + * @return the subscriberKey + */ + public String getSubscriberKey() { + return subscriberKey; + } + + /** + * @param subscriberKey the subscriberKey to set + */ + public void setSubscriberKey(String subscriberKey) { + this.subscriberKey = subscriberKey; + } + + /** + * @return the eventDate + */ + public Date getEventDate() { + return eventDate; + } + + /** + * @param eventDate the eventDate to set + */ + public void setEventDate(Date eventDate) { + this.eventDate = eventDate; + } + + /** + * @return the eventType + */ + public EventType getEventType() { + return eventType; + } + + /** + * @param eventType the eventType to set + */ + public void setEventType(EventType eventType) { + this.eventType = eventType; + } + + /** + * @return the triggeredSendDefinitionObjectID + */ + public String getTriggeredSendDefinitionObjectID() { + return triggeredSendDefinitionObjectID; + } + + /** + * @param triggeredSendDefinitionObjectID the triggeredSendDefinitionObjectID to set + */ + public void setTriggeredSendDefinitionObjectID(String triggeredSendDefinitionObjectID) { + this.triggeredSendDefinitionObjectID = triggeredSendDefinitionObjectID; + } + + /** + * @return the batchID + */ + public Integer getBatchID() { + return batchID; + } + + /** + * @param batchID the batchID to set + */ + public void setBatchID(Integer batchID) { + this.batchID = batchID; + } + + /** + * @return the urlid + */ + public Integer getUrlid() { + return urlid; + } + + /** + * @param urlid the urlid to set + */ + public void setUrlid(Integer urlid) { + this.urlid = urlid; + } + + /** + * @return the url + */ + public String getUrl() { + return url; + } + + /** + * @param url the url to set + */ + public void setUrl(String url) { + this.url = url; + } + +} diff --git a/src/main/java/com/custom/fuelsdk/ETNotSentEvent.java b/src/main/java/com/custom/fuelsdk/ETNotSentEvent.java new file mode 100644 index 0000000..8a21078 --- /dev/null +++ b/src/main/java/com/custom/fuelsdk/ETNotSentEvent.java @@ -0,0 +1,155 @@ +/* + * Copyright © 2020 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.custom.fuelsdk; + +import com.exacttarget.fuelsdk.ETSoapObject; +import com.exacttarget.fuelsdk.annotations.ExternalName; +import com.exacttarget.fuelsdk.annotations.InternalName; +import com.exacttarget.fuelsdk.annotations.SoapObject; +import com.exacttarget.fuelsdk.internal.EventType; +import com.exacttarget.fuelsdk.internal.NotSentEvent; + +import java.util.Date; + +/** + * An ETNotSentEvent object represents information pertaining to the notsent events + * of an email message in the Salesforce Marketing Cloud. + * This oject type is currently not supported in FuelSDK and hence the provided here. + */ +@SoapObject(internalType = NotSentEvent.class) +public class ETNotSentEvent extends ETSoapObject { + @InternalName("objectID") + private String id; + + @ExternalName("sendID") + private Integer sendID; + + @ExternalName("subscriberKey") + private String subscriberKey; + + @ExternalName("eventDate") + private Date eventDate; + + @ExternalName("eventType") + private EventType eventType; + + @ExternalName("triggeredSendDefinitionObjectID") + private String triggeredSendDefinitionObjectID; + + @ExternalName("batchID") + private Integer batchID; + + /** + * @return the id + */ + @Override + public String getId() { + return id; + } + + /** + * @param id the id to set + */ + @Override + public void setId(String id) { + this.id = id; + } + + /** + * @return the sendID + */ + public Integer getSendID() { + return sendID; + } + + /** + * @param sendID the sendID to set + */ + public void setSendID(Integer sendID) { + this.sendID = sendID; + } + + /** + * @return the subscriberKey + */ + public String getSubscriberKey() { + return subscriberKey; + } + + /** + * @param subscriberKey the subscriberKey to set + */ + public void setSubscriberKey(String subscriberKey) { + this.subscriberKey = subscriberKey; + } + + /** + * @return the eventDate + */ + public Date getEventDate() { + return eventDate; + } + + /** + * @param eventDate the eventDate to set + */ + public void setEventDate(Date eventDate) { + this.eventDate = eventDate; + } + + /** + * @return the eventType + */ + public EventType getEventType() { + return eventType; + } + + /** + * @param eventType the eventType to set + */ + public void setEventType(EventType eventType) { + this.eventType = eventType; + } + + /** + * @return the triggeredSendDefinitionObjectID + */ + public String getTriggeredSendDefinitionObjectID() { + return triggeredSendDefinitionObjectID; + } + + /** + * @param triggeredSendDefinitionObjectID the triggeredSendDefinitionObjectID to set + */ + public void setTriggeredSendDefinitionObjectID(String triggeredSendDefinitionObjectID) { + this.triggeredSendDefinitionObjectID = triggeredSendDefinitionObjectID; + } + + /** + * @return the batchID + */ + public Integer getBatchID() { + return batchID; + } + + /** + * @param batchID the batchID to set + */ + public void setBatchID(Integer batchID) { + this.batchID = batchID; + } + +} diff --git a/src/main/java/com/custom/fuelsdk/PaginationETSoapObject.java b/src/main/java/com/custom/fuelsdk/PaginationETSoapObject.java new file mode 100644 index 0000000..5741260 --- /dev/null +++ b/src/main/java/com/custom/fuelsdk/PaginationETSoapObject.java @@ -0,0 +1,366 @@ +/* + * Copyright © 2020 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.custom.fuelsdk; + +import com.exacttarget.fuelsdk.ETClient; +import com.exacttarget.fuelsdk.ETDataExtensionColumn; +import com.exacttarget.fuelsdk.ETDataExtensionRow; +import com.exacttarget.fuelsdk.ETExpression; +import com.exacttarget.fuelsdk.ETFilter; +import com.exacttarget.fuelsdk.ETResponse; +import com.exacttarget.fuelsdk.ETResult; +import com.exacttarget.fuelsdk.ETSdkException; +import com.exacttarget.fuelsdk.ETSoapConnection; +import com.exacttarget.fuelsdk.ETSoapObject; +import com.exacttarget.fuelsdk.annotations.SoapObject; +import com.exacttarget.fuelsdk.internal.APIObject; +import com.exacttarget.fuelsdk.internal.RetrieveRequest; +import com.exacttarget.fuelsdk.internal.RetrieveRequestMsg; +import com.exacttarget.fuelsdk.internal.RetrieveResponseMsg; +import com.exacttarget.fuelsdk.internal.Soap; +import org.apache.log4j.Logger; + +import java.util.ArrayList; +import java.util.List; + +import static com.exacttarget.fuelsdk.ETDataExtension.retrieveColumns; + +/** + * Extending class to fix the bug with Fuel-SDK on pagination. + */ +public class PaginationETSoapObject extends ETSoapObject { + + static final int DEFAULT_PAGE_SIZE = 2500; + private static Logger logger = Logger.getLogger(PaginationETSoapObject.class); + + /** + * @param client + * @param dataExtension + * @param filter + * @return + * @throws ETSdkException + */ + public static ETResponse select(ETClient client, + String dataExtension, + ETFilter filter) + throws ETSdkException { + String name = null; + + // + // The data extension can be specified using key or name: + // + + ETExpression e = ETExpression.parse(dataExtension); + if (e.getProperty().toLowerCase().equals("key") + && e.getOperator() == ETExpression.Operator.EQUALS) { + name = e.getValue(); + // if no columns are explicitly requested + // retrieve all columns + if (filter.getProperties().isEmpty()) { + filter.setProperties(retrieveColumnNames(client, name)); + } + } else if (e.getProperty().toLowerCase().equals("name") + && e.getOperator() == ETExpression.Operator.EQUALS) { + name = e.getValue(); + // if no columns are explicitly requested + // throw an exception + // because we need the key + // to retrieve columns + if (filter.getProperties().isEmpty()) { + throw new ETSdkException("columns must be specified " + + "when retrieving data extensions by name"); + } + + } else { + throw new ETSdkException("invalid data extension filter string"); + } + + ETResponse response = customRetrieve(client, "DataExtensionObject[" + name + "]", + filter, null, ETDataExtensionRow.class); + return response; + } + + /** + * @param client + * @param soapObject + * @param requestID + * @param filter + * @return + * @throws ETSdkException + */ + public static ETResponse continueRequest(ETClient client, String soapObject, String requestID, + ETFilter filter) throws ETSdkException { + ETResponse response = customRetrieve(client, soapObject, filter, requestID, + ETDataExtensionRow.class); + logger.debug("final response: " + response); + return response; + } + + //there was a bug with original method related with continueRequest + public static ETResponse customRetrieve(ETClient client, + String soapObjectName, + ETFilter filter, + String continueRequest, + Class type) + throws ETSdkException { + ETResponse response = new ETResponse(); + + // + // Get handle to the SOAP connection: + // + + ETSoapConnection connection = client.getSoapConnection(); + + // + // Automatically refresh the token if necessary: + // + + client.refreshToken(); + + // + // Read internal type from the SoapObject annotation: + // + + Class externalType = type; // for code readability + + SoapObject internalTypeAnnotation + = externalType.getAnnotation(SoapObject.class); + assert internalTypeAnnotation != null; + Class internalType = internalTypeAnnotation.internalType(); + assert internalType != null; + + ETExpression expression = filter.getExpression(); + + // + // Determine properties to retrieve: + // + + List externalProperties = filter.getProperties(); + List internalProperties = null; + + if (externalProperties.size() > 0) { + // + // Only request those properties specified: + // + + internalProperties = new ArrayList(); + + for (String externalProperty : externalProperties) { + String internalProperty = + getInternalProperty(externalType, externalProperty); + assert internalProperty != null; + internalProperties.add(internalProperty); + } + } else { + // + // No properties were explicitly requested: + // + + internalProperties = getInternalProperties(externalType); + + // + // Remove properties that are unretrievable: + // + + for (String property : internalTypeAnnotation.unretrievable()) { + internalProperties.remove(property); + } + } + + // + // Perform the SOAP retrieve: + // + + //Soap soap = connection.getSoap(); + Soap soap = null; + + RetrieveRequest retrieveRequest = new RetrieveRequest(); + + if (continueRequest == null) { + // if soapObjectType is specified, use it; otherwise, default + // to the name of the internal class representing the object: + if (soapObjectName != null) { + retrieveRequest.setObjectType(soapObjectName); + soap = connection.getSoap("retrieve", soapObjectName); + } else { + retrieveRequest.setObjectType(internalType.getSimpleName()); + soap = connection.getSoap("retrieve", internalType.getSimpleName()); + } + retrieveRequest.getProperties().addAll(internalProperties); + + if (expression.getOperator() != null) { + // + // Convert the property names to their internal counterparts: + // + + String property = expression.getProperty(); + if (property != null) { + expression.setProperty(getInternalProperty(type, property)); + } + for (ETExpression subexpression : expression.getSubexpressions()) { + String p = subexpression.getProperty(); + if (p != null) { + subexpression.setProperty(getInternalProperty(type, p)); + } + } + + retrieveRequest.setFilter(toFilterPart(expression)); + } + } else { + if (continueRequest != null) { + retrieveRequest.setContinueRequest(continueRequest); + } + } + +// //todo reduce batchsize +// RetrieveOptions retrieveOptions = new RetrieveOptions(); +//// retrieveOptions.setBatchSize(1); +// retrieveRequest.setOptions(retrieveOptions); + + if (logger.isTraceEnabled()) { + logger.trace("RetrieveRequest:"); + logger.trace(" objectType = " + retrieveRequest.getObjectType()); + StringBuilder line = new StringBuilder(" properties = { "); + //String line = null; + for (String property : retrieveRequest.getProperties()) { + + line.append(property).append(", "); + } + if (retrieveRequest.getProperties().size() > 0) { + line.setLength(line.length() - 2); + /*if (line == null) { + line = " properties = { " + property; + } else { + line += ", " + property; + }*/ + } + line.append(" }"); + logger.trace(line.toString()); + //logger.trace(line + " }"); + if (filter != null) { + logger.trace(" filter = " + toFilterPart(expression)); + } + } + + logger.trace("calling soap.retrieve..."); + + RetrieveRequestMsg retrieveRequestMsg = new RetrieveRequestMsg(); + retrieveRequestMsg.setRetrieveRequest(retrieveRequest); + + //todo temp fix for continue request + if (soap == null) { + if (soapObjectName != null) { +// retrieveRequest.setObjectType(soapObjectName); + soap = connection.getSoap("retrieve", soapObjectName); + } else { +// retrieveRequest.setObjectType(internalType.getSimpleName()); + soap = connection.getSoap("retrieve", internalType.getSimpleName()); + } + } + RetrieveResponseMsg retrieveResponseMsg = soap.retrieve(retrieveRequestMsg); + + if (logger.isTraceEnabled()) { + logger.trace("RetrieveResponseMsg:"); + logger.trace(" requestId = " + retrieveResponseMsg.getRequestID()); + logger.trace(" overallStatus = " + retrieveResponseMsg.getOverallStatus()); + logger.trace(" results = {"); + for (APIObject result : retrieveResponseMsg.getResults()) { + logger.trace(" " + result); + } + logger.trace(" }"); + } + + response.setRequestId(retrieveResponseMsg.getRequestID()); + if (retrieveResponseMsg.getOverallStatus().equals("OK")) { + response.setStatus(ETResult.Status.OK); + } else if (retrieveResponseMsg.getOverallStatus().equals("Error")) { + response.setStatus(ETResult.Status.ERROR); + } + response.setResponseCode(retrieveResponseMsg.getOverallStatus()); + response.setResponseMessage(retrieveResponseMsg.getOverallStatus()); + for (APIObject internalObject : retrieveResponseMsg.getResults()) { + // + // Allocate a new (external) object: + // + + T externalObject = null; + try { + externalObject = externalType.newInstance(); + } catch (Exception ex) { + throw new ETSdkException("could not instantiate " + + externalType.getName(), ex); + } + + externalObject.setClient(client); + + // + // Convert from internal representation: + // + + externalObject.fromInternal(internalObject); + + // + // Add result to the list of results: + // + + ETResult result = new ETResult(); + result.setObject(externalObject); + response.addResult(result); + } + + if (retrieveResponseMsg.getOverallStatus().equals("MoreDataAvailable")) { + response.setMoreResults(true); + } + + return response; + } + + /** + * @param client The ETClient object + * @param key The key + * @return The List of Column names + * @throws ETSdkException + */ + private static List retrieveColumnNames(ETClient client, + String key) + throws ETSdkException { + return getColumnNames(retrieveColumns(client, key)); + } + + + /** + * @param columns The List of ETDataExtensionColumn object + * @return The List of ETDataExtension Column names + */ + private static List getColumnNames(List columns) { + List columnNames = new ArrayList(); + for (ETDataExtensionColumn column : columns) { + columnNames.add(column.getName()); + } + return columnNames; + } + + @Override + public String getId() { + return null; + } + + @Override + public void setId(String s) { + + } +} diff --git a/src/main/java/com/custom/fuelsdk/internal/ClickEvent.java b/src/main/java/com/custom/fuelsdk/internal/ClickEvent.java new file mode 100644 index 0000000..19b0b7f --- /dev/null +++ b/src/main/java/com/custom/fuelsdk/internal/ClickEvent.java @@ -0,0 +1,133 @@ +/* + * Copyright © 2020 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.custom.fuelsdk.internal; + +import com.exacttarget.fuelsdk.internal.TrackingEvent; +import org.apache.commons.lang.builder.ToStringBuilder; +import org.apache.cxf.xjc.runtime.JAXBToStringStyle; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlType; + + +/** + *

Java class for ClickEvent complex type. + * + *

The following schema fragment specifies the expected content contained within this class. + * + * 

+ * <complexType name="ClickEvent">
+ *   <complexContent>
+ *     <extension base="{http://exacttarget.com/wsdl/partnerAPI}TrackingEvent">
+ *       <sequence>
+ *         <element name="URLID" type="{http://www.w3.org/2001/XMLSchema}int" minOccurs="0"/>
+ *         <element name="URL" type="{http://www.w3.org/2001/XMLSchema}string" minOccurs="0"/>
+ *         <element name="URLIDLong" type="{http://www.w3.org/2001/XMLSchema}long" minOccurs="0"/>
+ *       </sequence>
+ *     </extension>
+ *   </complexContent>
+ * </complexType>
+ *
+ */ +@XmlAccessorType(XmlAccessType.FIELD) +@XmlType(name = "ClickEvent", propOrder = { + "urlid", + "url", + "urlIdLong" +}) +public class ClickEvent + extends TrackingEvent { + + @XmlElement(name = "URLID") + protected Integer urlid; + @XmlElement(name = "URL") + protected String url; + @XmlElement(name = "URLIDLong") + protected Long urlIdLong; + + /** + * Gets the value of the urlid property. + * + * @return possible object is + * {@link Integer } + */ + public Integer getUrlid() { + return urlid; + } + + /** + * Sets the value of the urlid property. + * + * @param value allowed object is + * {@link Integer } + */ + public void setUrlid(Integer value) { + this.urlid = value; + } + + /** + * Gets the value of the url property. + * + * @return possible object is + * {@link String } + */ + public String getUrl() { + return url; + } + + /** + * Sets the value of the url property. + * + * @param value allowed object is + * {@link String } + */ + public void setUrl(String value) { + this.url = value; + } + + /** + * Gets the value of the urlIdLong property. + * + * @return possible object is + * {@link Long } + */ + public Long getUrlIdLong() { + return urlIdLong; + } + + /** + * Sets the value of the urlIdLong property. + * + * @param value allowed object is + * {@link Long } + */ + public void setUrlIdLong(Long value) { + this.urlIdLong = value; + } + + /** + * Generates a String representation of the contents of this type. + * This is an extension method, produced by the 'ts' xjc plugin + */ + @Override + public String toString() { + return ToStringBuilder.reflectionToString(this, JAXBToStringStyle.DEFAULT_STYLE); + } + +} diff --git a/src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudClient.java b/src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudClient.java new file mode 100644 index 0000000..f1e6867 --- /dev/null +++ b/src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudClient.java @@ -0,0 +1,241 @@ +/* + * Copyright © 2020 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.sfmc.source; + +//import com.exacttarget.fuelsdk.ETApiObject; + +import com.custom.fuelsdk.PaginationETSoapObject; +import com.exacttarget.fuelsdk.ETClient; +import com.exacttarget.fuelsdk.ETConfiguration; +import com.exacttarget.fuelsdk.ETDataExtensionColumn; +import com.exacttarget.fuelsdk.ETDataExtensionRow; +import com.exacttarget.fuelsdk.ETExpression; +import com.exacttarget.fuelsdk.ETFilter; +import com.exacttarget.fuelsdk.ETResponse; +import com.exacttarget.fuelsdk.ETSdkException; +import com.exacttarget.fuelsdk.ETSoapObject; +import io.cdap.plugin.sfmc.source.util.MarketingCloudColumn; +import io.cdap.plugin.sfmc.source.util.MarketingCloudObjectInfo; +import io.cdap.plugin.sfmc.source.util.SourceObject; +import io.cdap.plugin.sfmc.source.util.Util; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Utility class that provides methods to connect to Salesforce instance. + */ +public class MarketingCloudClient { + private final ETClient client; + private static final Logger LOG = LoggerFactory.getLogger(MarketingCloudClient.class); + + MarketingCloudClient(ETClient client) { + this.client = client; + } + + /** + * Initialize the connection with Salesforce Marketing Cloud using FuelSDK. + * + * @param clientId The Salesforce Marketing Cloud Client Id + * @param clientSecret The Salesforce Marketing Cloud Client Secret + * @param authEndpoint Auth Endpoint url for Salesforce Marketing Cloud + * @param soapEndpoint SOAP Endpoint url for Salesforce Marketing Cloud + * @return The instance of MarketingCloudClient object + * @throws ETSdkException The FuelSDKException + */ + public static MarketingCloudClient create(String clientId, String clientSecret, String authEndpoint, + String soapEndpoint) throws ETSdkException { + ETConfiguration conf = new ETConfiguration(); + conf.set("clientId", clientId); + conf.set("clientSecret", clientSecret); + conf.set("authEndpoint", authEndpoint); + conf.set("soapEndpoint", soapEndpoint); + conf.set("useOAuth2Authentication", "true"); + ClassLoader oldCL = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(MarketingCloudClient.class.getClassLoader()); + return new MarketingCloudClient(new ETClient(conf)); + } finally { + Thread.currentThread().setContextClassLoader(oldCL); + } + } + + /** + * Check if filter string is a valid SFMC Expression. + * + * @param filter + * @return + */ + public static void validateFilter(String filter) throws ETSdkException { + if (!Util.isNullOrEmpty(filter)) { + ETExpression expression = ETExpression.parse(filter); + } + } + + private ETExpression getExpressionfromString(String filter) throws ETSdkException { + if (!Util.isNullOrEmpty(filter)) { + return ETExpression.parse(filter); + } else { + return new ETExpression(); + } + } + + /** + * Fetch records for passed object from Salesforce Marketing Cloud. + * + * @param object The SourceObject which tells what data to be fetched from Salesforce Marketing Cloud + * @return The list of ETApiObject representing the records from requested object + */ + public List fetchObjectRecords(SourceObject object) throws ETSdkException { + ETFilter filter = new ETFilter(); + filter.setExpression(getExpressionfromString(object.getFilter())); + return fetchObjectData(client, object.getClassRef(), filter); + } + + /** + * Fetch the schema information for passed object from Salesforce Marketing Cloud. + * + * @param object The SourceObject which tells for which schema to be fetched from Salesforce Marketing Cloud + * @return The instance of MarketingCloudObjectInfo object + * @throws ETSdkException The FuelSDKException + */ + public MarketingCloudObjectInfo fetchObjectSchema(SourceObject object) + throws ETSdkException { + Class clazz = object.getClassRef(); + return new MarketingCloudObjectInfo(object, fetchObjectFields(clazz)); + } + + /** + * Fetch records for passed object from Salesforce Marketing Cloud. + * + * @param dataExtensionKey The data extension key for which data to be fetched from Salesforce Marketing Cloud + * @return The list of ETDataExtensionRow representing the records from requested data extension + * @throws ETSdkException The FuelSDKException + */ + public List fetchDataExtensionRecords(String dataExtensionKey, String filterStr) + throws ETSdkException { + + ETFilter filter = new ETFilter(); + filter.setExpression(getExpressionfromString(filterStr)); + + ETResponse response = PaginationETSoapObject.select(client, "key=" + dataExtensionKey, + filter); + List rows = response.getObjects(); + while (response.getResponseMessage().equals("MoreDataAvailable")) { + response = PaginationETSoapObject.continueRequest(client, null, response.getRequestId(), + filter); + rows.addAll(response.getObjects()); + } + return rows; + //return call(() -> ETDataExtension.select(client, "key=" + dataExtensionKey) + //.getObjects()); + } + + /** + * Fetch the schema information for passed object from Salesforce Marketing Cloud. + * + * @param dataExtensionKey The data extension key for which schema to be fetched from Salesforce Marketing Cloud + * @return The instance of MarketingCloudObjectInfo object + * @throws ETSdkException The FuelSDKException + */ + public MarketingCloudObjectInfo fetchDataExtensionSchema(String dataExtensionKey) + throws ETSdkException { + return call(() -> { + ETExpression expression = buildDataExtensionExpression(dataExtensionKey); + + ETFilter filter = new ETFilter(); + filter.setExpression(expression); + filter.addProperty("name"); + filter.addProperty("type"); + + ETResponse response = ETDataExtensionColumn.retrieve(client, ETDataExtensionColumn.class, + (Integer) null, (Integer) null, + filter); + List columns = response.getObjects().stream() + .map(o -> new MarketingCloudColumn(o.getName(), o.getType().name())) + .collect(Collectors.toList()); + + //Integer recordCount = fetchDataExtensionRecordCount(dataExtensionKey); + + return new MarketingCloudObjectInfo(SourceObject.DATA_EXTENSION, dataExtensionKey, columns); + }); + } + + private ETExpression buildDataExtensionExpression(String dataExtensionKey) { + ETExpression expression = new ETExpression(); + expression.setProperty("DataExtension.CustomerKey"); + expression.setOperator(ETExpression.Operator.EQUALS); + expression.addValue(dataExtensionKey); + return expression; + } + + private List fetchObjectData(ETClient client, Class clazz, ETFilter filter) + throws ETSdkException { + + ClassLoader oldCL = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(MarketingCloudClient.class.getClassLoader()); + //ETResponse etResponse = client.retrieve(clazz, new ETFilter()); + ETResponse response = (ETResponse) PaginationETSoapObject.customRetrieve(client, null, + filter, null, + clazz); + List rows = response.getObjects(); + while (response.getResponseMessage().equals("MoreDataAvailable")) { + response = (ETResponse) PaginationETSoapObject.customRetrieve(client, null, + filter, response.getRequestId(), + clazz); + rows.addAll(response.getObjects()); + } + return rows; + } finally { + Thread.currentThread().setContextClassLoader(oldCL); + } + } + + + private List fetchObjectFields(Class clazz) { + return Arrays.stream(clazz.getDeclaredFields()) + .map(o -> new MarketingCloudColumn(o.getName(), o.getType().getSimpleName().toUpperCase())) + .collect(Collectors.toList()); + } + + private T call(SFMCCall callable) throws ETSdkException { + ClassLoader oldClassloader = Thread.currentThread().getContextClassLoader(); + try { + Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); + return callable.call(); + } finally { + Thread.currentThread().setContextClassLoader(oldClassloader); + } + } + + /** + * A SFMC call. + * + * @param type of return object + */ + private interface SFMCCall { + + /** + * Perform a call. + */ + T call() throws ETSdkException; + } +} diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java b/src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudInputFormat.java similarity index 58% rename from src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java rename to src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudInputFormat.java index e5f823d..a3d9eff 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputFormat.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudInputFormat.java @@ -17,8 +17,7 @@ package io.cdap.plugin.sfmc.source; import io.cdap.cdap.api.data.format.StructuredRecord; -import io.cdap.plugin.sfmc.source.util.SalesforceConstants; -import io.cdap.plugin.sfmc.source.util.SalesforceObjectInfo; +import io.cdap.plugin.sfmc.source.util.MarketingCloudObjectInfo; import io.cdap.plugin.sfmc.source.util.SourceObject; import io.cdap.plugin.sfmc.source.util.SourceQueryMode; import io.cdap.plugin.sfmc.source.util.Util; @@ -40,8 +39,8 @@ /** * Salesforce input format. */ -public class SalesforceInputFormat extends InputFormat { - private static final Logger LOG = LoggerFactory.getLogger(SalesforceInputFormat.class); +public class MarketingCloudInputFormat extends InputFormat { + private static final Logger LOG = LoggerFactory.getLogger(MarketingCloudInputFormat.class); /** * Configure the input format to read tables from Salesforce. Should be called from the mapreduce client. @@ -49,16 +48,15 @@ public class SalesforceInputFormat extends InputFormat setInput(Configuration jobConfig, SourceQueryMode mode, - SalesforceSourceConfig conf, boolean fetchRecordCount) { - SalesforceJobConfiguration jobConf = new SalesforceJobConfiguration(jobConfig); + public static List setInput(Configuration jobConfig, SourceQueryMode mode, + MarketingCloudSourceConfig conf) { + MarketingCloudJobConfiguration jobConf = new MarketingCloudJobConfiguration(jobConfig); jobConf.setPluginConfiguration(conf); - //Depending on the selected objects in the conf, get the schema for each object as SalesforceObjectInfo - List tableInfos = fetchTableInfo(mode, conf, fetchRecordCount); + //Depending on the selected objects in the conf, get the schema for each object as MarketingCloudObjectInfo + List tableInfos = fetchTableInfo(mode, conf); jobConf.setTableInfos(tableInfos); @@ -71,45 +69,43 @@ public static List setInput(Configuration jobConfig, Sourc * * @param mode the query mode * @param conf the plugin conf - * @param fetchRecordCount the flag to decide whether to fetch record count or not - * @return Collection of SalesforceObjectInfo containing table and schema. + * @return Collection of MarketingCloudObjectInfo containing table and schema. */ - private static List fetchTableInfo(SourceQueryMode mode, SalesforceSourceConfig conf, - boolean fetchRecordCount) { + private static List fetchTableInfo(SourceQueryMode mode, MarketingCloudSourceConfig conf) { try { - SalesforceClient client = SalesforceClient.create(conf.getClientId(), conf.getClientSecret(), - conf.getAuthEndpoint(), conf.getSoapEndpoint()); + MarketingCloudClient client = MarketingCloudClient.create(conf.getClientId(), conf.getClientSecret(), + conf.getAuthEndpoint(), conf.getSoapEndpoint()); boolean failOnError = conf.isFailOnError(); //When mode = SingleObject, fetch fields for the object selected in plugin config if (mode == SourceQueryMode.SINGLE_OBJECT) { - SalesforceObjectInfo tableInfo = getTableMetaData(conf.getObject(), conf.getDataExtensionKey(), client, - failOnError, fetchRecordCount); + MarketingCloudObjectInfo tableInfo = getTableMetaData(conf.getObject(), conf.getDataExtensionKey(), client, + failOnError); return (tableInfo == null) ? Collections.emptyList() : Collections.singletonList(tableInfo); } //When mode = MultiObject, get the list of objects provided in plugin config and the fetch fields for each of //then objects. - List tableInfos = new ArrayList<>(); + List tableInfos = new ArrayList<>(); List objectList = conf.getObjectList(); for (SourceObject object : objectList) { - SalesforceObjectInfo tableInfo = null; + MarketingCloudObjectInfo tableInfo = null; if (object == SourceObject.DATA_EXTENSION) { //if the object = Data Extension then get the list of data extension keys and then fetch fields for each of //the data extension objects. List dataExtensionKeys = Util.splitToList(conf.getDataExtensionKeys(), ','); for (String dataExtensionKey : dataExtensionKeys) { - tableInfo = getTableMetaData(object, dataExtensionKey, client, failOnError, fetchRecordCount); + tableInfo = getTableMetaData(object, dataExtensionKey, client, failOnError); if (tableInfo == null) { continue; } tableInfos.add(tableInfo); } } else { - tableInfo = getTableMetaData(object, "", client, failOnError, fetchRecordCount); + tableInfo = getTableMetaData(object, "", client, failOnError); if (tableInfo != null) { tableInfos.add(tableInfo); } @@ -130,14 +126,13 @@ private static List fetchTableInfo(SourceQueryMode mode, S /** * Fetch the fields for passed object. */ - private static SalesforceObjectInfo getTableMetaData(SourceObject object, String dataExtensionKey, - SalesforceClient client, boolean failOnError, - boolean fetchRecordCount) { + private static MarketingCloudObjectInfo getTableMetaData(SourceObject object, String dataExtensionKey, + MarketingCloudClient client, boolean failOnError) { try { if (object == SourceObject.DATA_EXTENSION) { - return client.fetchDataExtensionSchema(dataExtensionKey, fetchRecordCount); + return client.fetchDataExtensionSchema(dataExtensionKey); } else { - return client.fetchObjectSchema(object, fetchRecordCount); + return client.fetchObjectSchema(object); } } catch (Exception e) { if (failOnError) { @@ -151,26 +146,16 @@ private static SalesforceObjectInfo getTableMetaData(SourceObject object, String @Override public List getSplits(JobContext jobContext) throws IOException, InterruptedException { - SalesforceJobConfiguration jobConfig = new SalesforceJobConfiguration(jobContext.getConfiguration()); - SalesforceSourceConfig pluginConf = jobConfig.getPluginConf(); + MarketingCloudJobConfiguration jobConfig = new MarketingCloudJobConfiguration(jobContext.getConfiguration()); + MarketingCloudSourceConfig pluginConf = jobConfig.getPluginConf(); - List tableInfos = jobConfig.getTableInfos(); + List tableInfos = jobConfig.getTableInfos(); List resultSplits = new ArrayList<>(); - for (SalesforceObjectInfo tableInfo : tableInfos) { + for (MarketingCloudObjectInfo tableInfo : tableInfos) { String tableKey = tableInfo.getObject().name(); String tableName = tableInfo.getTableName(); - int totalRecords = tableInfo.getRecordCount(); - if (totalRecords <= SalesforceConstants.MAX_PAGE_SIZE) { - //add single split for table and continue - resultSplits.add(new SalesforceInputSplit(tableKey, tableName, 1, totalRecords)); - continue; - } - - int pages = (tableInfo.getRecordCount() / SalesforceConstants.MAX_PAGE_SIZE) + 1; - for (int page = 1; page <= pages; page++) { - resultSplits.add(new SalesforceInputSplit(tableKey, tableName, page, SalesforceConstants.MAX_PAGE_SIZE)); - } + resultSplits.add(new MarketingCloudInputSplit(tableKey, tableName)); } LOG.debug("# of split = {}", resultSplits.size()); @@ -182,9 +167,10 @@ public List getSplits(JobContext jobContext) throws IOException, Int public RecordReader createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { - SalesforceJobConfiguration jobConfig = new SalesforceJobConfiguration(taskAttemptContext.getConfiguration()); - SalesforceSourceConfig pluginConf = jobConfig.getPluginConf(); + MarketingCloudJobConfiguration jobConfig = new MarketingCloudJobConfiguration( + taskAttemptContext.getConfiguration()); + MarketingCloudSourceConfig pluginConf = jobConfig.getPluginConf(); - return new SalesforceRecordReader(pluginConf); + return new MarketingCloudRecordReader(pluginConf); } } diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputSplit.java b/src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudInputSplit.java similarity index 72% rename from src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputSplit.java rename to src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudInputSplit.java index ec24f61..396409e 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceInputSplit.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudInputSplit.java @@ -26,29 +26,23 @@ /** * Salesforce input split. */ -public class SalesforceInputSplit extends InputSplit implements Writable { +public class MarketingCloudInputSplit extends InputSplit implements Writable { private String objectName; private String tableName; - private int page; - private int pageSize; // used by mapreduce - public SalesforceInputSplit() { + public MarketingCloudInputSplit() { } /** - * Constructor for SalesforceInputSplit. + * Constructor for MarketingCloudInputSplit. * * @param objectName The object name * @param tableName The corresponding table name - * @param page The page index - * @param pageSize The page size */ - public SalesforceInputSplit(String objectName, String tableName, int page, int pageSize) { + public MarketingCloudInputSplit(String objectName, String tableName) { this.objectName = objectName; this.tableName = tableName; - this.page = page; - this.pageSize = pageSize; } public String getObjectName() { @@ -59,28 +53,17 @@ public String getTableName() { return tableName; } - public int getPage() { - return page; - } - - public int getPageSize() { - return pageSize; - } @Override public void write(DataOutput dataOutput) throws IOException { dataOutput.writeUTF(this.objectName); dataOutput.writeUTF(this.tableName); - dataOutput.writeInt(this.page); - dataOutput.writeInt(this.pageSize); } @Override public void readFields(DataInput dataInput) throws IOException { this.objectName = dataInput.readUTF(); this.tableName = dataInput.readUTF(); - this.page = dataInput.readInt(); - this.pageSize = dataInput.readInt(); } @Override diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceJobConfiguration.java b/src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudJobConfiguration.java similarity index 72% rename from src/main/java/io/cdap/plugin/sfmc/source/SalesforceJobConfiguration.java rename to src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudJobConfiguration.java index 27c707d..42d42cf 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceJobConfiguration.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudJobConfiguration.java @@ -18,7 +18,7 @@ import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; -import io.cdap.plugin.sfmc.source.util.SalesforceObjectInfo; +import io.cdap.plugin.sfmc.source.util.MarketingCloudObjectInfo; import org.apache.hadoop.conf.Configuration; import java.lang.reflect.Type; @@ -27,20 +27,20 @@ /** * Allows to specify and access connection configuration properties of {@link Configuration}. */ -public class SalesforceJobConfiguration { +public class MarketingCloudJobConfiguration { private static final String PLUGIN_CONF_FIELD = "salesforce.plugin.conf"; private static final String INFO_FIELD = "salesforce.table.info"; - private static final Type PLUGIN_CONF_TYPE = new TypeToken() { + private static final Type PLUGIN_CONF_TYPE = new TypeToken() { }.getType(); - private static final Type INFO_TYPE = new TypeToken>() { + private static final Type INFO_TYPE = new TypeToken>() { }.getType(); private static final Gson GSON = new Gson(); private Configuration conf; - public SalesforceJobConfiguration(Configuration job) { + public MarketingCloudJobConfiguration(Configuration job) { this.conf = job; } @@ -48,19 +48,19 @@ public Configuration getConf() { return this.conf; } - public void setPluginConfiguration(SalesforceSourceConfig conf) { + public void setPluginConfiguration(MarketingCloudSourceConfig conf) { set(PLUGIN_CONF_FIELD, GSON.toJson(conf)); } - public SalesforceSourceConfig getPluginConf() { + public MarketingCloudSourceConfig getPluginConf() { return GSON.fromJson(getConf().get(PLUGIN_CONF_FIELD), PLUGIN_CONF_TYPE); } - public List getTableInfos() { + public List getTableInfos() { return GSON.fromJson(getConf().get(INFO_FIELD), INFO_TYPE); } - public void setTableInfos(List infoList) { + public void setTableInfos(List infoList) { set(INFO_FIELD, GSON.toJson(infoList)); } diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java b/src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudRecordReader.java similarity index 87% rename from src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java rename to src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudRecordReader.java index 056e51e..365b8fa 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceRecordReader.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudRecordReader.java @@ -21,8 +21,8 @@ import com.google.common.base.Strings; import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; -import io.cdap.plugin.sfmc.source.util.SalesforceConstants; -import io.cdap.plugin.sfmc.source.util.SalesforceObjectInfo; +import io.cdap.plugin.sfmc.source.util.MarketingCloudConstants; +import io.cdap.plugin.sfmc.source.util.MarketingCloudObjectInfo; import io.cdap.plugin.sfmc.source.util.SourceObject; import io.cdap.plugin.sfmc.source.util.SourceQueryMode; import org.apache.hadoop.io.NullWritable; @@ -42,13 +42,13 @@ /** * Record reader that reads the entire contents of a Salesforce table. */ -public class SalesforceRecordReader extends RecordReader { - private static final Logger LOG = LoggerFactory.getLogger(SalesforceRecordReader.class); - private final SalesforceSourceConfig pluginConf; - private SalesforceInputSplit split; +public class MarketingCloudRecordReader extends RecordReader { + private static final Logger LOG = LoggerFactory.getLogger(MarketingCloudRecordReader.class); + private final MarketingCloudSourceConfig pluginConf; + private MarketingCloudInputSplit split; private int pos; private List tableFields; - private SalesforceObjectInfo sfObjectMetaData; + private MarketingCloudObjectInfo sfObjectMetaData; private Schema schema; private SourceObject object; @@ -59,13 +59,13 @@ public class SalesforceRecordReader extends RecordReader iterator; private ETApiObject row; - SalesforceRecordReader(SalesforceSourceConfig pluginConf) { + MarketingCloudRecordReader(MarketingCloudSourceConfig pluginConf) { this.pluginConf = pluginConf; } @Override public void initialize(InputSplit split, TaskAttemptContext context) { - this.split = (SalesforceInputSplit) split; + this.split = (MarketingCloudInputSplit) split; this.pos = 0; } @@ -135,20 +135,21 @@ private void fetchData() { object = SourceObject.valueOf(split.getObjectName()); tableName = split.getTableName(); if (object == SourceObject.DATA_EXTENSION) { - dataExtensionKey = tableName.replaceAll(SalesforceConstants.DATA_EXTENSION_PREFIX, ""); + dataExtensionKey = tableName.replaceAll(MarketingCloudConstants.DATA_EXTENSION_PREFIX, ""); } tableNameField = pluginConf.getTableNameField(); //Get the table data try { - SalesforceClient client = SalesforceClient.create(pluginConf.getClientId(), - pluginConf.getClientSecret(), pluginConf.getAuthEndpoint(), pluginConf.getSoapEndpoint()); + MarketingCloudClient client = MarketingCloudClient.create(pluginConf.getClientId(), pluginConf.getClientSecret(), + pluginConf.getAuthEndpoint(), + pluginConf.getSoapEndpoint()); //Fetch data if (object == SourceObject.DATA_EXTENSION) { - results = client.fetchDataExtensionRecords(dataExtensionKey, split.getPage(), split.getPageSize()); + results = client.fetchDataExtensionRecords(dataExtensionKey, object.getFilter()); } else { - results = client.fetchObjectRecords(object, split.getPage(), split.getPageSize()); + results = client.fetchObjectRecords(object); } LOG.debug("size={}", results.size()); @@ -167,15 +168,15 @@ private void fetchData() { iterator = results.iterator(); } - private void fetchSchema(SalesforceClient client) { + private void fetchSchema(MarketingCloudClient client) { //Fetch the column definition List schemaFields; try { if (object == SourceObject.DATA_EXTENSION) { - sfObjectMetaData = client.fetchDataExtensionSchema(dataExtensionKey, false); + sfObjectMetaData = client.fetchDataExtensionSchema(dataExtensionKey); } else { - sfObjectMetaData = client.fetchObjectSchema(object, false); + sfObjectMetaData = client.fetchObjectSchema(object); } //Build schema diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java b/src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudSource.java similarity index 81% rename from src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java rename to src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudSource.java index d4cded5..9f0b5ec 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSource.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudSource.java @@ -33,9 +33,8 @@ import io.cdap.cdap.etl.api.batch.BatchSourceContext; import io.cdap.plugin.common.LineageRecorder; import io.cdap.plugin.common.SourceInputFormatProvider; -import io.cdap.plugin.sfmc.source.util.SalesforceConstants; -import io.cdap.plugin.sfmc.source.util.SalesforceObjectInfo; -import io.cdap.plugin.sfmc.source.util.SourceObject; +import io.cdap.plugin.sfmc.source.util.MarketingCloudConstants; +import io.cdap.plugin.sfmc.source.util.MarketingCloudObjectInfo; import io.cdap.plugin.sfmc.source.util.SourceQueryMode; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.NullWritable; @@ -54,16 +53,16 @@ * A {@link BatchSource} that reads data from multiple objects in Salesforce. */ @Plugin(type = BatchSource.PLUGIN_TYPE) -@Name(SalesforceConstants.PLUGIN_NAME) +@Name(MarketingCloudConstants.PLUGIN_NAME) @Description("Reads from multiple objects in Salesforce. " + "Outputs one record for each row in each object, with the object name as a record field. " + "Also sets a pipeline argument for each object read, which contains the object schema.") -public class SalesforceSource extends BatchSource { - private static final Logger LOG = LoggerFactory.getLogger(SalesforceSource.class); +public class MarketingCloudSource extends BatchSource { + private static final Logger LOG = LoggerFactory.getLogger(MarketingCloudSource.class); - private final SalesforceSourceConfig conf; + private final MarketingCloudSourceConfig conf; - public SalesforceSource(SalesforceSourceConfig conf) { + public MarketingCloudSource(MarketingCloudSourceConfig conf) { this.conf = conf; } @@ -105,16 +104,16 @@ public void prepareRun(BatchSourceContext context) throws Exception { SourceQueryMode mode = conf.getQueryMode(collector); Configuration hConf = new Configuration(); - Collection tables = SalesforceInputFormat.setInput(hConf, mode, conf, true); + Collection tables = MarketingCloudInputFormat.setInput(hConf, mode, conf); SettableArguments arguments = context.getArguments(); - for (SalesforceObjectInfo tableInfo : tables) { - arguments.set(SalesforceConstants.TABLE_PREFIX + tableInfo.getFormattedTableName(), + for (MarketingCloudObjectInfo tableInfo : tables) { + arguments.set(MarketingCloudConstants.TABLE_PREFIX + tableInfo.getFormattedTableName(), tableInfo.getSchema().toString()); recordLineage(context, tableInfo); } context.setInput(Input.of(conf.getReferenceName(), - new SourceInputFormatProvider(SalesforceInputFormat.class, hConf))); + new SourceInputFormatProvider(MarketingCloudInputFormat.class, hConf))); } @Override @@ -125,9 +124,10 @@ public void transform(KeyValue input, Emitter tables = SalesforceInputFormat.setInput(hConf, mode, conf, false); + Collection tables = MarketingCloudInputFormat.setInput(hConf, mode, conf); if (tables != null && !tables.isEmpty()) { schema = tables.iterator().next().getSchema(); } @@ -136,7 +136,7 @@ private Schema getSchema(SourceQueryMode mode) { return schema; } - private void recordLineage(BatchSourceContext context, SalesforceObjectInfo tableInfo) { + private void recordLineage(BatchSourceContext context, MarketingCloudObjectInfo tableInfo) { String tableName = tableInfo.getFormattedTableName(); String outputName = String.format("%s-%s", conf.getReferenceName(), tableName); Schema schema = tableInfo.getSchema(); diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java b/src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudSourceConfig.java similarity index 64% rename from src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java rename to src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudSourceConfig.java index 5e58346..80c4a72 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfig.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudSourceConfig.java @@ -25,7 +25,7 @@ import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.plugin.common.Constants; import io.cdap.plugin.common.IdUtils; -import io.cdap.plugin.sfmc.source.util.SalesforceConstants; +import io.cdap.plugin.sfmc.source.util.MarketingCloudConstants; import io.cdap.plugin.sfmc.source.util.SourceObject; import io.cdap.plugin.sfmc.source.util.SourceQueryMode; import io.cdap.plugin.sfmc.source.util.Util; @@ -38,23 +38,23 @@ import javax.annotation.Nullable; /** - * Configuration for the {@link SalesforceSource}. + * Configuration for the {@link MarketingCloudSource}. */ -public class SalesforceSourceConfig extends PluginConfig { - private static final Logger LOG = LoggerFactory.getLogger(SalesforceSourceConfig.class); +public class MarketingCloudSourceConfig extends PluginConfig { + private static final Logger LOG = LoggerFactory.getLogger(MarketingCloudSourceConfig.class); @Name(Constants.Reference.REFERENCE_NAME) @Description("This will be used to uniquely identify this source for lineage, annotating metadata, etc.") private String referenceName; - @Name(SalesforceConstants.PROPERTY_QUERY_MODE) + @Name(MarketingCloudConstants.PROPERTY_QUERY_MODE) @Macro @Description("Mode of data retrieval. The mode can be one of two values: " + "`Multi Object` - will allow user to fetch data for multiple data extensions, " + "`Single Object` - will allow user to fetch data for single data extension.") private String queryMode; - @Name(SalesforceConstants.PROPERTY_OBJECT_NAME) + @Name(MarketingCloudConstants.PROPERTY_OBJECT_NAME) @Macro @Nullable @Description("Specify the object for which data to be fetched. This can be one of following values: " + @@ -65,7 +65,7 @@ public class SalesforceSourceConfig extends PluginConfig { "Note, this value will be ignored if the Mode is set to `Multi Object`.") private String objectName; - @Name(SalesforceConstants.PROPERTY_DATA_EXTENSION_KEY) + @Name(MarketingCloudConstants.PROPERTY_DATA_EXTENSION_KEY) @Macro @Nullable @Description("Specify the data extension key from which data to be fetched. Note, this value will be ignored in " + @@ -73,7 +73,7 @@ public class SalesforceSourceConfig extends PluginConfig { "`Data Extension`.") private String dataExtensionKey; - @Name(SalesforceConstants.PROPERTY_OBJECT_LIST) + @Name(MarketingCloudConstants.PROPERTY_OBJECT_LIST) @Macro @Nullable @Description("Specify the comma-separated list of objects for which data to be fetched; for example: " + @@ -85,7 +85,7 @@ public class SalesforceSourceConfig extends PluginConfig { "Note, this value will be ignored if the Mode is set to `Single Object`.") private String objectList; - @Name(SalesforceConstants.PROPERTY_DATA_EXTENSION_KEY_LIST) + @Name(MarketingCloudConstants.PROPERTY_DATA_EXTENSION_KEY_LIST) @Macro @Nullable @Description("Specify the data extension keys from which data to be fetched; for example: 'Key1,Key2'. " + @@ -93,7 +93,7 @@ public class SalesforceSourceConfig extends PluginConfig { "2. If the selected object list does not contain `Data Extension` as one of the objects.") private String dataExtensionKeys; - @Name(SalesforceConstants.PROPERTY_TABLE_NAME_FIELD) + @Name(MarketingCloudConstants.PROPERTY_TABLE_NAME_FIELD) @Macro @Nullable @Description("The name of the field that holds the object name to which the data belongs to. Must not be the name " + @@ -102,29 +102,35 @@ public class SalesforceSourceConfig extends PluginConfig { "value will be ignored if the Mode is set to `Single Object`.") private String tableNameField; - @Name(SalesforceConstants.PROPERTY_CLIENT_ID) + @Name(MarketingCloudConstants.PROPERTY_FILTER) + @Macro + @Nullable + @Description("The WHERE clause used to filter data from Marketing cloud objects.") + private String filter; + + @Name(MarketingCloudConstants.PROPERTY_CLIENT_ID) @Macro @Description("OAuth2 client ID associated with an installed package in the Salesforce Marketing Cloud.") private String clientId; - @Name(SalesforceConstants.PROPERTY_CLIENT_SECRET) + @Name(MarketingCloudConstants.PROPERTY_CLIENT_SECRET) @Macro @Description("OAuth2 client secret associated with an installed package in the Salesforce Marketing Cloud.") private String clientSecret; - @Name(SalesforceConstants.PROPERTY_API_ENDPOINT) + @Name(MarketingCloudConstants.PROPERTY_API_ENDPOINT) @Macro @Description("The REST API Base URL associated for the Server-to-Server API integration. " + "For example, https://instance.rest.marketingcloudapis.com/") private String restEndpoint; - @Name(SalesforceConstants.PROPERTY_AUTH_API_ENDPOINT) + @Name(MarketingCloudConstants.PROPERTY_AUTH_API_ENDPOINT) @Macro @Description("Authentication Base URL associated for the Server-to-Server API integration. " + "For example, https://instance.auth.marketingcloudapis.com/") private String authEndpoint; - @Name(SalesforceConstants.PROPERTY_SOAP_API_ENDPOINT) + @Name(MarketingCloudConstants.PROPERTY_SOAP_API_ENDPOINT) @Macro @Description("The SOAP Endpoint URL associated for the Server-to-Server API integration. " + "For example, https://instance.soap.marketingcloudapis.com/Service.asmx") @@ -132,32 +138,33 @@ public class SalesforceSourceConfig extends PluginConfig { @Macro @Nullable - @Name(SalesforceConstants.PROPERTY_FAIL_ON_ERROR) + @Name(MarketingCloudConstants.PROPERTY_FAIL_ON_ERROR) @Description("Whether to fail the pipeline if it fails while reading.") private Boolean failOnError; /** - * Constructor for SalesforceSourceConfig object. + * Constructor for MarketingCloudSourceConfig object. * - * @param referenceName The reference name - * @param queryMode The query mode - * @param objectName The object name to be fetched from Salesforce Marketing Cloud - * @param dataExtensionKey The data extension key to be fetched from Salesforce Marketing Cloud - * @param objectList The list of objects to be fetched from Salesforce Marketing Cloud + * @param referenceName The reference name + * @param queryMode The query mode + * @param objectName The object name to be fetched from Salesforce Marketing Cloud + * @param dataExtensionKey The data extension key to be fetched from Salesforce Marketing Cloud + * @param objectList The list of objects to be fetched from Salesforce Marketing Cloud * @param dataExtensionKeys The list of data extension keys to be fetched from Salesforce Marketing Cloud - * @param tableNameField The field name to hold the table name value - * @param clientId The Salesforce Marketing Cloud Client Id - * @param clientSecret The Salesforce Marketing Cloud Client Secret - * @param restEndpoint The REST API endpoint for Salesforce Marketing Cloud - * @param authEndpoint The AUTH API endpoint for Salesforce Marketing Cloud - * @param soapEndpoint The SOAP API endpoint for Salesforce Marketing Cloud - * @param failOnError A flag to decide what is to be done to the pipeline execution in case of errors + * @param tableNameField The field name to hold the table name value + * @param clientId The Salesforce Marketing Cloud Client Id + * @param clientSecret The Salesforce Marketing Cloud Client Secret + * @param restEndpoint The REST API endpoint for Salesforce Marketing Cloud + * @param authEndpoint The AUTH API endpoint for Salesforce Marketing Cloud + * @param soapEndpoint The SOAP API endpoint for Salesforce Marketing Cloud + * @param failOnError A flag to decide what is to be done to the pipeline execution in case of errors */ - public SalesforceSourceConfig(String referenceName, String queryMode, @Nullable String objectName, - @Nullable String dataExtensionKey, @Nullable String objectList, - @Nullable String dataExtensionKeys, @Nullable String tableNameField, String clientId, - String clientSecret, String restEndpoint, String authEndpoint, String soapEndpoint, - Boolean failOnError) { + public MarketingCloudSourceConfig(String referenceName, String queryMode, @Nullable String objectName, + @Nullable String dataExtensionKey, @Nullable String objectList, + @Nullable String dataExtensionKeys, @Nullable String tableNameField, + @Nullable String filter, String clientId, String clientSecret, + String restEndpoint, String authEndpoint, String soapEndpoint, + Boolean failOnError) { this.referenceName = referenceName; this.queryMode = queryMode; this.objectName = objectName; @@ -165,6 +172,7 @@ public SalesforceSourceConfig(String referenceName, String queryMode, @Nullable this.objectList = objectList; this.dataExtensionKeys = dataExtensionKeys; this.tableNameField = tableNameField; + this.filter = filter; this.clientId = clientId; this.clientSecret = clientSecret; this.restEndpoint = restEndpoint; @@ -190,8 +198,8 @@ public SourceQueryMode getQueryMode(FailureCollector collector) { } collector.addFailure("Unsupported query mode value: " + queryMode, - String.format("Supported modes are: %s", SourceQueryMode.getSupportedModes())) - .withConfigProperty(SalesforceConstants.PROPERTY_QUERY_MODE); + String.format("Supported modes are: %s", SourceQueryMode.getSupportedModes())) + .withConfigProperty(MarketingCloudConstants.PROPERTY_QUERY_MODE); collector.getOrThrowException(); return null; } @@ -220,15 +228,15 @@ public SourceObject getObject(FailureCollector collector) { } collector.addFailure("Unsupported object value: " + objectName, - String.format("Supported objects are: %s", SourceObject.getSupportedObjects())) - .withConfigProperty(SalesforceConstants.PROPERTY_OBJECT_NAME); + String.format("Supported objects are: %s", SourceObject.getSupportedObjects())) + .withConfigProperty(MarketingCloudConstants.PROPERTY_OBJECT_NAME); collector.getOrThrowException(); return null; } @Nullable public SourceObject getObject() { - return getSourceObject(objectName); + return getSourceObject(objectName, filter); } @Nullable @@ -247,14 +255,13 @@ public List getObjectList(FailureCollector collector) { List sourceObjects = new ArrayList<>(); for (String object : objects) { - SourceObject sourceObject = getSourceObject(object); + SourceObject sourceObject = getSourceObject(object, filter); if (sourceObject == null) { collector.addFailure("Unsupported object value: " + object, - String.format("Supported objects are: %s", SourceObject.getSupportedObjects())) - .withConfigProperty(SalesforceConstants.PROPERTY_OBJECT_LIST); + String.format("Supported objects are: %s", SourceObject.getSupportedObjects())) + .withConfigProperty(MarketingCloudConstants.PROPERTY_OBJECT_LIST); break; } - sourceObjects.add(sourceObject); } @@ -272,11 +279,10 @@ public List getObjectList() { List sourceObjects = new ArrayList<>(); for (String object : objects) { - SourceObject sourceObject = getSourceObject(object); + SourceObject sourceObject = getSourceObject(object, filter); if (sourceObject == null) { continue; } - sourceObjects.add(sourceObject); } @@ -293,6 +299,11 @@ public String getTableNameField() { return tableNameField; } + @Nullable + public String getFilter() { + return filter; + } + public String getClientId() { return clientId; } @@ -318,7 +329,7 @@ public Boolean isFailOnError() { } /** - * Validates {@link SalesforceSourceConfig} instance. + * Validates {@link MarketingCloudSourceConfig} instance. */ public void validate(FailureCollector collector) { //Validates the given referenceName to consists of characters allowed to represent a dataset. @@ -326,12 +337,18 @@ public void validate(FailureCollector collector) { validateCredentials(collector); validateQueryMode(collector); + validateFilter(collector); } - private SourceObject getSourceObject(String objectName) { + private SourceObject getSourceObject(String objectName, String filter) { Optional sourceObject = SourceObject.fromValue(objectName); - - return sourceObject.isPresent() ? sourceObject.get() : null; + if (sourceObject.isPresent()) { + SourceObject obj = sourceObject.get(); + obj.setFilter(filter); + return obj; + } else { + return null; + } } private void validateCredentials(FailureCollector collector) { @@ -341,27 +358,27 @@ private void validateCredentials(FailureCollector collector) { if (Util.isNullOrEmpty(clientId)) { collector.addFailure("Client ID must be specified.", null) - .withConfigProperty(SalesforceConstants.PROPERTY_CLIENT_ID); + .withConfigProperty(MarketingCloudConstants.PROPERTY_CLIENT_ID); } if (Util.isNullOrEmpty(clientSecret)) { collector.addFailure("Client Secret must be specified.", null) - .withConfigProperty(SalesforceConstants.PROPERTY_CLIENT_SECRET); + .withConfigProperty(MarketingCloudConstants.PROPERTY_CLIENT_SECRET); } if (Util.isNullOrEmpty(restEndpoint)) { collector.addFailure(" REST Endpoint must be specified.", null) - .withConfigProperty(SalesforceConstants.PROPERTY_API_ENDPOINT); + .withConfigProperty(MarketingCloudConstants.PROPERTY_API_ENDPOINT); } if (Util.isNullOrEmpty(authEndpoint)) { collector.addFailure("Auth Endpoint must be specified.", null) - .withConfigProperty(SalesforceConstants.PROPERTY_AUTH_API_ENDPOINT); + .withConfigProperty(MarketingCloudConstants.PROPERTY_AUTH_API_ENDPOINT); } if (Util.isNullOrEmpty(soapEndpoint)) { collector.addFailure("Soap Endpoint must be specified.", null) - .withConfigProperty(SalesforceConstants.PROPERTY_SOAP_API_ENDPOINT); + .withConfigProperty(MarketingCloudConstants.PROPERTY_SOAP_API_ENDPOINT); } collector.getOrThrowException(); @@ -371,23 +388,23 @@ private void validateCredentials(FailureCollector collector) { @VisibleForTesting void validateSalesforceConnection(FailureCollector collector) { try { - SalesforceClient.create(clientId, clientSecret, authEndpoint, soapEndpoint); + MarketingCloudClient.create(clientId, clientSecret, authEndpoint, soapEndpoint); } catch (ETSdkException e) { collector.addFailure("Unable to connect to Salesforce Instance.", - "Ensure properties like Client ID, Client Secret, API Endpoint, Soap Endpoint, Auth Endpoint " + - "are correct.") - .withConfigProperty(SalesforceConstants.PROPERTY_CLIENT_ID) - .withConfigProperty(SalesforceConstants.PROPERTY_CLIENT_SECRET) - .withConfigProperty(SalesforceConstants.PROPERTY_API_ENDPOINT) - .withConfigProperty(SalesforceConstants.PROPERTY_AUTH_API_ENDPOINT) - .withConfigProperty(SalesforceConstants.PROPERTY_SOAP_API_ENDPOINT) + "Ensure properties like Client ID, Client Secret, API Endpoint " + + ", Soap Endpoint, Auth Endpoint are correct.") + .withConfigProperty(MarketingCloudConstants.PROPERTY_CLIENT_ID) + .withConfigProperty(MarketingCloudConstants.PROPERTY_CLIENT_SECRET) + .withConfigProperty(MarketingCloudConstants.PROPERTY_API_ENDPOINT) + .withConfigProperty(MarketingCloudConstants.PROPERTY_AUTH_API_ENDPOINT) + .withConfigProperty(MarketingCloudConstants.PROPERTY_SOAP_API_ENDPOINT) .withStacktrace(e.getStackTrace()); } } private void validateQueryMode(FailureCollector collector) { //according to query mode check if either object name / object list exists or not - if (containsMacro(SalesforceConstants.PROPERTY_QUERY_MODE)) { + if (containsMacro(MarketingCloudConstants.PROPERTY_QUERY_MODE)) { return; } @@ -401,9 +418,9 @@ private void validateQueryMode(FailureCollector collector) { } private void validateMultiObjectQueryMode(FailureCollector collector) { - if (containsMacro(SalesforceConstants.PROPERTY_OBJECT_LIST) - || containsMacro(SalesforceConstants.PROPERTY_DATA_EXTENSION_KEY_LIST) - || containsMacro(SalesforceConstants.PROPERTY_TABLE_NAME_FIELD)) { + if (containsMacro(MarketingCloudConstants.PROPERTY_OBJECT_LIST) + || containsMacro(MarketingCloudConstants.PROPERTY_DATA_EXTENSION_KEY_LIST) + || containsMacro(MarketingCloudConstants.PROPERTY_TABLE_NAME_FIELD)) { return; } @@ -412,26 +429,26 @@ private void validateMultiObjectQueryMode(FailureCollector collector) { if (objects.isEmpty()) { collector.addFailure("At least 1 Object must be specified.", null) - .withConfigProperty(SalesforceConstants.PROPERTY_OBJECT_LIST); + .withConfigProperty(MarketingCloudConstants.PROPERTY_OBJECT_LIST); } if (objects.contains(SourceObject.DATA_EXTENSION)) { List dataExtensionKeyList = Util.splitToList(getDataExtensionKeys(), ','); if (dataExtensionKeyList.isEmpty()) { collector.addFailure("At least 1 Data Extension Key must be specified.", null) - .withConfigProperty(SalesforceConstants.PROPERTY_DATA_EXTENSION_KEY_LIST); + .withConfigProperty(MarketingCloudConstants.PROPERTY_DATA_EXTENSION_KEY_LIST); } } if (Util.isNullOrEmpty(tableNameField)) { collector.addFailure("Table name field must be specified.", null) - .withConfigProperty(SalesforceConstants.PROPERTY_TABLE_NAME_FIELD); + .withConfigProperty(MarketingCloudConstants.PROPERTY_TABLE_NAME_FIELD); } } private void validateSingleObjectQueryMode(FailureCollector collector) { - if (containsMacro(SalesforceConstants.PROPERTY_OBJECT_NAME) - || containsMacro(SalesforceConstants.PROPERTY_DATA_EXTENSION_KEY)) { + if (containsMacro(MarketingCloudConstants.PROPERTY_OBJECT_NAME) + || containsMacro(MarketingCloudConstants.PROPERTY_DATA_EXTENSION_KEY)) { return; } @@ -439,7 +456,21 @@ private void validateSingleObjectQueryMode(FailureCollector collector) { if (object == SourceObject.DATA_EXTENSION && Util.isNullOrEmpty(dataExtensionKey)) { collector.addFailure("Data Extension Key must be specified.", null) - .withConfigProperty(SalesforceConstants.PROPERTY_DATA_EXTENSION_KEY); + .withConfigProperty(MarketingCloudConstants.PROPERTY_DATA_EXTENSION_KEY); + } + } + + private void validateFilter(FailureCollector collector) { + if (containsMacro(MarketingCloudConstants.PROPERTY_FILTER) || Util.isNullOrEmpty(filter)) { + return; + } + try { + MarketingCloudClient.validateFilter(filter); + } catch (ETSdkException e) { + collector.addFailure("Filter string is not valid.", + "Check syntax to confirm.") + .withConfigProperty(MarketingCloudConstants.PROPERTY_FILTER) + .withStacktrace(e.getStackTrace()); } } @@ -447,10 +478,10 @@ private void validateSingleObjectQueryMode(FailureCollector collector) { * Returns true if Salesforce can be connected to. */ public boolean shouldConnect() { - return !containsMacro(SalesforceConstants.PROPERTY_CLIENT_ID) && - !containsMacro(SalesforceConstants.PROPERTY_CLIENT_SECRET) && - !containsMacro(SalesforceConstants.PROPERTY_API_ENDPOINT) && - !containsMacro(SalesforceConstants.PROPERTY_AUTH_API_ENDPOINT) && - !containsMacro(SalesforceConstants.PROPERTY_SOAP_API_ENDPOINT); + return !containsMacro(MarketingCloudConstants.PROPERTY_CLIENT_ID) && + !containsMacro(MarketingCloudConstants.PROPERTY_CLIENT_SECRET) && + !containsMacro(MarketingCloudConstants.PROPERTY_API_ENDPOINT) && + !containsMacro(MarketingCloudConstants.PROPERTY_AUTH_API_ENDPOINT) && + !containsMacro(MarketingCloudConstants.PROPERTY_SOAP_API_ENDPOINT); } } diff --git a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceClient.java b/src/main/java/io/cdap/plugin/sfmc/source/SalesforceClient.java deleted file mode 100644 index 7155fd4..0000000 --- a/src/main/java/io/cdap/plugin/sfmc/source/SalesforceClient.java +++ /dev/null @@ -1,225 +0,0 @@ -/* - * Copyright © 2020 Cask Data, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package io.cdap.plugin.sfmc.source; - -import com.exacttarget.fuelsdk.ETApiObject; -import com.exacttarget.fuelsdk.ETClient; -import com.exacttarget.fuelsdk.ETConfiguration; -import com.exacttarget.fuelsdk.ETDataExtension; -import com.exacttarget.fuelsdk.ETDataExtensionColumn; -import com.exacttarget.fuelsdk.ETDataExtensionRow; -import com.exacttarget.fuelsdk.ETExpression; -import com.exacttarget.fuelsdk.ETFilter; -import com.exacttarget.fuelsdk.ETResponse; -import com.exacttarget.fuelsdk.ETSdkException; -import io.cdap.plugin.sfmc.source.util.SalesforceColumn; -import io.cdap.plugin.sfmc.source.util.SalesforceConstants; -import io.cdap.plugin.sfmc.source.util.SalesforceObjectInfo; -import io.cdap.plugin.sfmc.source.util.SourceObject; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; - -/** - * Utility class that provides methods to connect to Salesforce instance. - */ -public class SalesforceClient { - private final ETClient client; - - SalesforceClient(ETClient client) { - this.client = client; - } - - /** - * Initialize the connection with Salesforce Marketing Cloud using FuelSDK. - * - * @param clientId The Salesforce Marketing Cloud Client Id - * @param clientSecret The Salesforce Marketing Cloud Client Secret - * @param authEndpoint Auth Endpoint url for Salesforce Marketing Cloud - * @param soapEndpoint SOAP Endpoint url for Salesforce Marketing Cloud - * @return The instance of SalesforceClient object - * @throws ETSdkException The FuelSDKException - */ - public static SalesforceClient create(String clientId, String clientSecret, String authEndpoint, - String soapEndpoint) throws ETSdkException { - ETConfiguration conf = new ETConfiguration(); - conf.set("clientId", clientId); - conf.set("clientSecret", clientSecret); - conf.set("authEndpoint", authEndpoint); - conf.set("soapEndpoint", soapEndpoint); - conf.set("useOAuth2Authentication", "true"); - ClassLoader oldCL = Thread.currentThread().getContextClassLoader(); - try { - Thread.currentThread().setContextClassLoader(SalesforceClient.class.getClassLoader()); - return new SalesforceClient(new ETClient(conf)); - } finally { - Thread.currentThread().setContextClassLoader(oldCL); - } - } - - /** - * Fetch records for passed object from Salesforce Marketing Cloud. - * - * @param object The SourceObject which tells what data to be fetched from Salesforce Marketing Cloud - * @param page The page index - * @param pageSize The size of the page - * @return The list of ETApiObject representing the records from requested object - */ - public List fetchObjectRecords(SourceObject object, int page, int pageSize) { - try { - Integer pageIndex = null; - Integer maxPageSize = null; - - if (object.isPagingSupported()) { - pageIndex = page; - maxPageSize = pageSize; - } - - return fetchObjectData(client, object.getClassRef(), pageIndex, maxPageSize); - } catch (ETSdkException e) { - return Collections.emptyList(); - } - } - - /** - * Fetch the schema information for passed object from Salesforce Marketing Cloud. - * - * @param object The SourceObject which tells for which schema to be fetched from Salesforce Marketing Cloud - * @param fetchRecordCount the flag to decide whether to fetch record count or not - * @return The instance of SalesforceObjectInfo object - * @throws ETSdkException The FuelSDKException - */ - public SalesforceObjectInfo fetchObjectSchema(SourceObject object, boolean fetchRecordCount) throws ETSdkException { - Class clazz = object.getClassRef(); - Integer recordCount = SalesforceConstants.MAX_PAGE_SIZE; - - if (object.isPagingSupported() && fetchRecordCount) { - ETResponse etResponse = client.retrieve(clazz, 1, 1, new ETFilter()); - recordCount = etResponse.getTotalCount(); - } - - return new SalesforceObjectInfo(object, fetchObjectFields(clazz), recordCount); - } - - /** - * Fetch records for passed object from Salesforce Marketing Cloud. - * - * @param dataExtensionKey The data extension key for which data to be fetched from Salesforce Marketing Cloud - * @param page The page index - * @param pageSize The size of the page - * @return The list of ETDataExtensionRow representing the records from requested data extension - * @throws ETSdkException The FuelSDKException - */ - public List fetchDataExtensionRecords(String dataExtensionKey, int page, int pageSize) - throws ETSdkException { - return call(() -> ETDataExtension.select(client, "key=" + dataExtensionKey, page, pageSize) - .getObjects()); - } - - /** - * Fetch the schema information for passed object from Salesforce Marketing Cloud. - * - * @param dataExtensionKey The data extension key for which schema to be fetched from Salesforce Marketing Cloud - * @param fetchRecordCount the flag to decide whether to fetch record count or not - * @return The instance of SalesforceObjectInfo object - * @throws ETSdkException The FuelSDKException - */ - public SalesforceObjectInfo fetchDataExtensionSchema(String dataExtensionKey, boolean fetchRecordCount) - throws ETSdkException { - return call(() -> { - ETExpression expression = buildDataExtensionExpression(dataExtensionKey); - - ETFilter filter = new ETFilter(); - filter.setExpression(expression); - filter.addProperty("name"); - filter.addProperty("type"); - - ETResponse response = ETDataExtensionColumn.retrieve(client, - ETDataExtensionColumn.class, (Integer) null, (Integer) null, filter); - List columns = response.getObjects().stream() - .map(o -> new SalesforceColumn(o.getName(), o.getType().name())) - .collect(Collectors.toList()); - - Integer recordCount = fetchDataExtensionRecordCount(dataExtensionKey); - - return new SalesforceObjectInfo(SourceObject.DATA_EXTENSION, dataExtensionKey, columns, recordCount); - }); - } - - private Integer fetchDataExtensionRecordCount(String dataExtensionKey) throws ETSdkException { - Integer totalCount = 0; - Integer pageRecordCount; - int page = 1; - - do { - pageRecordCount = ETDataExtension.select(client, "key=" + dataExtensionKey, page, - SalesforceConstants.MAX_PAGE_SIZE).getResults().size(); - totalCount += pageRecordCount; - page++; - } while (pageRecordCount >= SalesforceConstants.MAX_PAGE_SIZE); - - return totalCount; - } - - private ETExpression buildDataExtensionExpression(String dataExtensionKey) { - ETExpression expression = new ETExpression(); - - expression.setProperty("DataExtension.CustomerKey"); - expression.setOperator(ETExpression.Operator.EQUALS); - expression.addValue(dataExtensionKey); - - return expression; - } - - private List fetchObjectData(ETClient client, Class clazz, Integer page, - Integer pageSize) - throws ETSdkException { - ETResponse etResponse = client.retrieve(clazz, page, pageSize, new ETFilter()); - return etResponse.getObjects(); - } - - private List fetchObjectFields(Class clazz) { - return Arrays.stream(clazz.getDeclaredFields()) - .map(o -> new SalesforceColumn(o.getName(), o.getType().getSimpleName().toUpperCase())) - .collect(Collectors.toList()); - } - - private T call(SFMCCall callable) throws ETSdkException { - ClassLoader oldClassloader = Thread.currentThread().getContextClassLoader(); - try { - Thread.currentThread().setContextClassLoader(getClass().getClassLoader()); - return callable.call(); - } finally { - Thread.currentThread().setContextClassLoader(oldClassloader); - } - } - - /** - * A SFMC call. - * - * @param type of return object - */ - private interface SFMCCall { - - /** - * Perform a call. - */ - T call() throws ETSdkException; - } -} diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceColumn.java b/src/main/java/io/cdap/plugin/sfmc/source/util/MarketingCloudColumn.java similarity index 86% rename from src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceColumn.java rename to src/main/java/io/cdap/plugin/sfmc/source/util/MarketingCloudColumn.java index d4419bf..627c29b 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceColumn.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/MarketingCloudColumn.java @@ -19,23 +19,23 @@ import com.google.common.base.Strings; /** - * Information about a Salesforce table column. + * Information about a MarketingCloud table column. */ -public class SalesforceColumn { +public class MarketingCloudColumn { private String fieldName; private String typeName; private String mandatory; - public SalesforceColumn() { + public MarketingCloudColumn() { } /** - * Constructor for SalesforceColumn. + * Constructor for MarketingCloudColumn. * * @param fieldName The column name * @param typeName The data type name */ - public SalesforceColumn(String fieldName, String typeName) { + public MarketingCloudColumn(String fieldName, String typeName) { this.fieldName = fieldName; this.typeName = typeName; this.mandatory = "false"; diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java b/src/main/java/io/cdap/plugin/sfmc/source/util/MarketingCloudConstants.java similarity index 92% rename from src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java rename to src/main/java/io/cdap/plugin/sfmc/source/util/MarketingCloudConstants.java index 2125f7e..3b5f953 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceConstants.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/MarketingCloudConstants.java @@ -19,12 +19,12 @@ /** * Salesforce constants. */ -public interface SalesforceConstants { +public interface MarketingCloudConstants { /** * Salesforce plugin name. */ - String PLUGIN_NAME = "Salesforce"; + String PLUGIN_NAME = "MarketingCloud"; /** * Configuration property name used to specify the query mode. @@ -97,7 +97,8 @@ public interface SalesforceConstants { String DATA_EXTENSION_PREFIX = "dataextension_"; /** - * The max limit for the page size. + * Filter to be used when retrieving data from Salesforce Marketing Cloud. */ - int MAX_PAGE_SIZE = 2500; + String PROPERTY_FILTER = "filter"; + } diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceObjectInfo.java b/src/main/java/io/cdap/plugin/sfmc/source/util/MarketingCloudObjectInfo.java similarity index 73% rename from src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceObjectInfo.java rename to src/main/java/io/cdap/plugin/sfmc/source/util/MarketingCloudObjectInfo.java index a309153..20d5846 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SalesforceObjectInfo.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/MarketingCloudObjectInfo.java @@ -25,40 +25,38 @@ /** * Information about a Salesforce table. */ -public class SalesforceObjectInfo { +public class MarketingCloudObjectInfo { private final SourceObject object; private final String dataExtensionKey; private final Schema schema; private final Map mapSchemaFieldToSFMCField; - private final int recordCount; + /** - * Constructor for SalesforceObjectInfo for non-dataextension object. + * Constructor for MarketingCloudObjectInfo for non-dataextension object. * @param object The Salesforce Marketing Cloud object * @param columns The list of columns - * @param recordCount The total number of records */ - public SalesforceObjectInfo(SourceObject object, List columns, int recordCount) { - this(object, null, columns, recordCount); + public MarketingCloudObjectInfo(SourceObject object, List columns) { + this(object, null, columns); } /** - * Constructor for SalesforceObjectInfo for dataextension object. + * Constructor for MarketingCloudObjectInfo for dataextension object. * * @param object The Salesforce Marketing Cloud object as DataExtension * @param dataExtensionKey The data extension key * @param columns The list of columns - * @param recordCount The total number of records */ - public SalesforceObjectInfo(SourceObject object, String dataExtensionKey, List columns, - int recordCount) { + public MarketingCloudObjectInfo(SourceObject object, String dataExtensionKey, List columns + ) { this.object = object; this.dataExtensionKey = dataExtensionKey; SchemaBuilder schemaBuilder = new SchemaBuilder(); this.schema = schemaBuilder.constructSchema(getFormattedTableName(), columns); this.mapSchemaFieldToSFMCField = columns.stream() - .collect(Collectors.toMap(SalesforceColumn::getFormattedFieldName, SalesforceColumn::getFieldName)); - this.recordCount = recordCount; + .collect(Collectors.toMap(MarketingCloudColumn::getFormattedFieldName, MarketingCloudColumn::getFieldName)); + } public SourceObject getObject() { @@ -73,7 +71,7 @@ public SourceObject getObject() { */ public String getTableName() { if (getObject() == SourceObject.DATA_EXTENSION) { - return String.format("%s%s", SalesforceConstants.DATA_EXTENSION_PREFIX, dataExtensionKey); + return String.format("%s%s", MarketingCloudConstants.DATA_EXTENSION_PREFIX, dataExtensionKey); } else { return getObject().getTableName(); } @@ -91,9 +89,7 @@ public Schema getSchema() { return schema; } - public int getRecordCount() { - return recordCount; - } + public String lookupFieldsMap(String schemaFieldName) { if (this.mapSchemaFieldToSFMCField == null) { diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SchemaBuilder.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SchemaBuilder.java index d161a45..f2ac91e 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SchemaBuilder.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SchemaBuilder.java @@ -34,21 +34,21 @@ public class SchemaBuilder { * @param columns The list of SalesforceColumn objects that will be added as Schema.Field * @return The instance of Schema object */ - public Schema constructSchema(String tableName, List columns) { + public Schema constructSchema(String tableName, List columns) { SchemaBuilder schemaBuilder = new SchemaBuilder(); List fields = schemaBuilder.constructSchemaFields(columns); return Schema.recordOf(tableName, fields); } - private List constructSchemaFields(List columns) { + private List constructSchemaFields(List columns) { return columns.stream() .map(o -> transformToField(o)) .filter(Objects::nonNull) .collect(Collectors.toList()); } - private Schema.Field transformToField(SalesforceColumn column) { + private Schema.Field transformToField(MarketingCloudColumn column) { String name = column.getFormattedFieldName(); if (Strings.isNullOrEmpty(name)) { return null; @@ -64,7 +64,7 @@ private Schema.Field transformToField(SalesforceColumn column) { : Schema.Field.of(name, Schema.nullableOf(schema)); } - private Schema createSchema(SalesforceColumn column) { + private Schema createSchema(MarketingCloudColumn column) { switch (column.getTypeName().toLowerCase()) { case "decimal": case "double": diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SourceObject.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SourceObject.java index 30db38e..cf07501 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SourceObject.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SourceObject.java @@ -16,11 +16,15 @@ package io.cdap.plugin.sfmc.source.util; -import com.exacttarget.fuelsdk.ETApiObject; -import com.exacttarget.fuelsdk.ETCampaign; +import com.custom.fuelsdk.ETNotSentEvent; +import com.exacttarget.fuelsdk.ETBounceEvent; import com.exacttarget.fuelsdk.ETDataExtensionRow; import com.exacttarget.fuelsdk.ETEmail; import com.exacttarget.fuelsdk.ETList; +import com.exacttarget.fuelsdk.ETOpenEvent; +import com.exacttarget.fuelsdk.ETSentEvent; +import com.exacttarget.fuelsdk.ETSoapObject; +import com.exacttarget.fuelsdk.ETUnsubEvent; import java.util.Arrays; import java.util.Optional; @@ -28,7 +32,7 @@ import java.util.stream.Stream; /** - * Indicates the object for which data to be fetched from Salesforce. + * Indicates the object for which data to be fetched from MarketingCloud. */ public enum SourceObject { @@ -40,7 +44,7 @@ public enum SourceObject { /** * This indicates data to be fetched from Campaign. */ - CAMPAIGN("Campaign", "campaign", ETCampaign.class), +// CAMPAIGN("Campaign", "campaign", ETCampaign.class), /** * This indicates data to be fetched from Email. @@ -50,23 +54,48 @@ public enum SourceObject { /** * This indicates data to be fetched from Mailing List. */ - MAILING_LIST("Mailing List", "mailinglist", ETList.class); + MAILING_LIST("Mailing List", "mailinglist", ETList.class), + + /** + * This indicates data to be fetched from Tracking Bounce Events. + */ + TRACKING_BOUNCE_EVENT("Bounce Event", "bounce", ETBounceEvent.class), + + /** + * This indicates data to be fetched from Tracking Open Events. + */ + TRACKING_OPEN_EVENT("Open Event", "open", ETOpenEvent.class), + + /** + * This indicates data to be fetched from Tracking Click Events. + */ + //TRACKING_CLICK_EVENT("Click Event", "click", ETClickEvent.class), + + /** + * This indicates data to be fetched from Tracking UnSub Events. + */ + TRACKING_UNSUB_EVENT("Unsub Event", "unsub", ETUnsubEvent.class), + + /** + * This indicates data to be fetched from Tracking Sent Events. + */ + TRACKING_SENT_EVENT("Sent Event", "sent", ETSentEvent.class), + + /** + * This indicates data to be fetched from Tracking Notsent Events. + */ + TRACKING_NOTSENT_EVENT("Notsent Event", "notsent", ETNotSentEvent.class); private final String value; private final String tableName; - private final Class classRef; - private final boolean pagingSupported; + private final Class classRef; + private String filter = ""; + - SourceObject(String value, String tableName, Class classRef) { + SourceObject(String value, String tableName, Class classRef) { this.value = value; this.tableName = tableName; this.classRef = classRef; - if (tableName.equals("dataextension")) { - this.pagingSupported = true; - } else { - this.pagingSupported = getClassRef().getSuperclass() != null - && !getClassRef().getSuperclass().getSimpleName().equalsIgnoreCase("ETSoapObject"); - } } /** @@ -94,11 +123,16 @@ public String getTableName() { return tableName; } - public Class getClassRef() { + public Class getClassRef() { return classRef; } - public boolean isPagingSupported() { - return pagingSupported; + public String getFilter() { + return filter; } + + public void setFilter(String filter) { + this.filter = filter; + } + } diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SourceQueryMode.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SourceQueryMode.java index 512e804..f3f228a 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SourceQueryMode.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SourceQueryMode.java @@ -22,7 +22,7 @@ import java.util.stream.Stream; /** - * Indicates query mode which will be used when fetching Salesforce data. + * Indicates query mode which will be used when fetching MarketingCloud data. */ public enum SourceQueryMode { diff --git a/src/test/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfigHelper.java b/src/test/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfigHelper.java index 2317cbb..101ac51 100644 --- a/src/test/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfigHelper.java +++ b/src/test/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfigHelper.java @@ -29,6 +29,7 @@ public class SalesforceSourceConfigHelper { public static final String TEST_SOAP_ENDPOINT = "TestSoapEndpoint"; public static final String TEST_OBJECT_NAME = "Data Extension"; public static final String TEST_OBJECT_LIST = "Data Extension"; + private static final String TEST_FILTER = ""; public static ConfigBuilder newConfigBuilder() { return new ConfigBuilder(); @@ -43,6 +44,7 @@ public static class ConfigBuilder { private String soapEndpoint = TEST_SOAP_ENDPOINT; private String queryMode; private String objectName = TEST_OBJECT_NAME; + private String filter = TEST_FILTER; private String dataExtensionKey; private String objectList = TEST_OBJECT_LIST; private String dataExtensionKeys; @@ -63,6 +65,11 @@ public ConfigBuilder setObjectName(String objectName) { return this; } + public ConfigBuilder setFilter(String filter) { + this.filter = filter; + return this; + } + public ConfigBuilder setDataExtensionKey(String dataExtensionKey) { this.dataExtensionKey = dataExtensionKey; return this; @@ -108,10 +115,10 @@ public ConfigBuilder setSoapEndpoint(String soapEndpoint) { return this; } - public SalesforceSourceConfig build() { - return new SalesforceSourceConfig(referenceName, queryMode, objectName, dataExtensionKey, objectList, - dataExtensionKeys, tableNameField, clientId, clientSecret, restEndpoint, authEndpoint, soapEndpoint, - false); + public MarketingCloudSourceConfig build() { + return new MarketingCloudSourceConfig(referenceName, queryMode, objectName, dataExtensionKey, objectList, + dataExtensionKeys, tableNameField, filter, clientId, clientSecret, + restEndpoint, authEndpoint, soapEndpoint, false); } } } diff --git a/src/test/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfigTest.java b/src/test/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfigTest.java index 0bffbd5..f4d732f 100644 --- a/src/test/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfigTest.java +++ b/src/test/java/io/cdap/plugin/sfmc/source/SalesforceSourceConfigTest.java @@ -22,6 +22,7 @@ import io.cdap.cdap.etl.mock.validation.MockFailureCollector; import io.cdap.plugin.sfmc.source.util.SourceQueryMode; import org.junit.Assert; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -32,17 +33,17 @@ import static io.cdap.plugin.sfmc.source.SalesforceSourceConfigHelper.TEST_CLIENT_SECRET; import static io.cdap.plugin.sfmc.source.SalesforceSourceConfigHelper.TEST_REST_ENDPOINT; import static io.cdap.plugin.sfmc.source.SalesforceSourceConfigHelper.TEST_SOAP_ENDPOINT; -import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_AUTH_API_ENDPOINT; -import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_CLIENT_ID; -import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_CLIENT_SECRET; -import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_DATA_EXTENSION_KEY; -import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_DATA_EXTENSION_KEY_LIST; -import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_QUERY_MODE; -import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_SOAP_API_ENDPOINT; -import static io.cdap.plugin.sfmc.source.util.SalesforceConstants.PROPERTY_TABLE_NAME_FIELD; +import static io.cdap.plugin.sfmc.source.util.MarketingCloudConstants.PROPERTY_AUTH_API_ENDPOINT; +import static io.cdap.plugin.sfmc.source.util.MarketingCloudConstants.PROPERTY_CLIENT_ID; +import static io.cdap.plugin.sfmc.source.util.MarketingCloudConstants.PROPERTY_CLIENT_SECRET; +import static io.cdap.plugin.sfmc.source.util.MarketingCloudConstants.PROPERTY_DATA_EXTENSION_KEY; +import static io.cdap.plugin.sfmc.source.util.MarketingCloudConstants.PROPERTY_DATA_EXTENSION_KEY_LIST; +import static io.cdap.plugin.sfmc.source.util.MarketingCloudConstants.PROPERTY_QUERY_MODE; +import static io.cdap.plugin.sfmc.source.util.MarketingCloudConstants.PROPERTY_SOAP_API_ENDPOINT; +import static io.cdap.plugin.sfmc.source.util.MarketingCloudConstants.PROPERTY_TABLE_NAME_FIELD; /** - * Tests for {@link SalesforceSourceConfig}. + * Tests for {@link MarketingCloudSourceConfig}. */ public class SalesforceSourceConfigTest { @@ -52,7 +53,7 @@ public class SalesforceSourceConfigTest { @Test public void testQueryModeSingleObject() { SourceQueryMode queryMode = SourceQueryMode.SINGLE_OBJECT; - SalesforceSourceConfig config = SalesforceSourceConfigHelper.newConfigBuilder() + MarketingCloudSourceConfig config = SalesforceSourceConfigHelper.newConfigBuilder() .setQueryMode("Single Object") .build(); @@ -64,7 +65,7 @@ public void testQueryModeSingleObject() { @Test public void testQueryModeMultiObject() { SourceQueryMode queryMode = SourceQueryMode.MULTI_OBJECT; - SalesforceSourceConfig config = SalesforceSourceConfigHelper.newConfigBuilder() + MarketingCloudSourceConfig config = SalesforceSourceConfigHelper.newConfigBuilder() .setQueryMode("Multi Object") .build(); @@ -75,7 +76,7 @@ public void testQueryModeMultiObject() { @Test public void testQueryModeInvalid() { - SalesforceSourceConfig config = SalesforceSourceConfigHelper.newConfigBuilder() + MarketingCloudSourceConfig config = SalesforceSourceConfigHelper.newConfigBuilder() .setQueryMode(null) .build(); @@ -92,7 +93,7 @@ public void testQueryModeInvalid() { @Test public void testValidateClientIdNull() { MockFailureCollector collector = new MockFailureCollector(); - SalesforceSourceConfig config = withSalesforceValidationMock(SalesforceSourceConfigHelper.newConfigBuilder() + MarketingCloudSourceConfig config = withSalesforceValidationMock(SalesforceSourceConfigHelper.newConfigBuilder() .setQueryMode("Single Object") .setClientId(null) .setClientSecret(TEST_CLIENT_SECRET) @@ -114,7 +115,7 @@ public void testValidateClientIdNull() { @Test public void testValidateClientSecretNull() { MockFailureCollector collector = new MockFailureCollector(); - SalesforceSourceConfig config = withSalesforceValidationMock(SalesforceSourceConfigHelper.newConfigBuilder() + MarketingCloudSourceConfig config = withSalesforceValidationMock(SalesforceSourceConfigHelper.newConfigBuilder() .setQueryMode("Single Object") .setClientId(TEST_CLIENT_ID) .setClientSecret(null) @@ -136,7 +137,7 @@ public void testValidateClientSecretNull() { @Test public void testValidateAuthEndpointNull() { MockFailureCollector collector = new MockFailureCollector(); - SalesforceSourceConfig config = withSalesforceValidationMock(SalesforceSourceConfigHelper.newConfigBuilder() + MarketingCloudSourceConfig config = withSalesforceValidationMock(SalesforceSourceConfigHelper.newConfigBuilder() .setQueryMode("Single Object") .setClientId(TEST_CLIENT_ID) .setClientSecret(TEST_CLIENT_SECRET) @@ -158,7 +159,7 @@ public void testValidateAuthEndpointNull() { @Test public void testValidateSoapEndpointNull() { MockFailureCollector collector = new MockFailureCollector(); - SalesforceSourceConfig config = withSalesforceValidationMock(SalesforceSourceConfigHelper.newConfigBuilder() + MarketingCloudSourceConfig config = withSalesforceValidationMock(SalesforceSourceConfigHelper.newConfigBuilder() .setQueryMode("Single Object") .setClientId(TEST_CLIENT_ID) .setClientSecret(TEST_CLIENT_SECRET) @@ -178,9 +179,10 @@ public void testValidateSoapEndpointNull() { } @Test + @Ignore public void testValidCredentials() { MockFailureCollector collector = new MockFailureCollector(); - SalesforceSourceConfig config = withSalesforceValidationMock(SalesforceSourceConfigHelper.newConfigBuilder() + MarketingCloudSourceConfig config = withSalesforceValidationMock(SalesforceSourceConfigHelper.newConfigBuilder() .setQueryMode("Single Object") .setClientId(TEST_CLIENT_ID) .setClientSecret(TEST_CLIENT_SECRET) @@ -198,7 +200,7 @@ public void testValidCredentials() { @Test public void testSingleObjectModeMissingDataExtensionKey() { MockFailureCollector collector = new MockFailureCollector(); - SalesforceSourceConfig config = withSalesforceValidationMock(SalesforceSourceConfigHelper.newConfigBuilder() + MarketingCloudSourceConfig config = withSalesforceValidationMock(SalesforceSourceConfigHelper.newConfigBuilder() .setQueryMode("Single Object") .setRestEndpoint(TEST_REST_ENDPOINT) .setAuthEndpoint(TEST_AUTH_ENDPOINT) @@ -219,7 +221,7 @@ public void testSingleObjectModeMissingDataExtensionKey() { @Test public void testMultiObjectModeMissingDataExtensionKeys() { MockFailureCollector collector = new MockFailureCollector(); - SalesforceSourceConfig config = withSalesforceValidationMock(SalesforceSourceConfigHelper.newConfigBuilder() + MarketingCloudSourceConfig config = withSalesforceValidationMock(SalesforceSourceConfigHelper.newConfigBuilder() .setQueryMode("Multi Object") .setClientId(TEST_CLIENT_ID) .setClientSecret(TEST_CLIENT_SECRET) @@ -242,7 +244,7 @@ public void testMultiObjectModeMissingDataExtensionKeys() { @Test public void testMultiObjectModeMissingTableNameField() { MockFailureCollector collector = new MockFailureCollector(); - SalesforceSourceConfig config = withSalesforceValidationMock(SalesforceSourceConfigHelper.newConfigBuilder() + MarketingCloudSourceConfig config = withSalesforceValidationMock(SalesforceSourceConfigHelper.newConfigBuilder() .setQueryMode("Multi Object") .setClientId(TEST_CLIENT_ID) .setClientSecret(TEST_CLIENT_SECRET) @@ -263,9 +265,9 @@ public void testMultiObjectModeMissingTableNameField() { } } - private SalesforceSourceConfig withSalesforceValidationMock(SalesforceSourceConfig config, - FailureCollector collector) { - SalesforceSourceConfig spy = Mockito.spy(config); + private MarketingCloudSourceConfig withSalesforceValidationMock(MarketingCloudSourceConfig config, + FailureCollector collector) { + MarketingCloudSourceConfig spy = Mockito.spy(config); Mockito.doNothing().when(spy).validateSalesforceConnection(collector); return spy; } diff --git a/widgets/Salesforce-batchsource.json b/widgets/MarketingCloud-batchsource.json similarity index 86% rename from widgets/Salesforce-batchsource.json rename to widgets/MarketingCloud-batchsource.json index 12501b7..358e53f 100644 --- a/widgets/Salesforce-batchsource.json +++ b/widgets/MarketingCloud-batchsource.json @@ -38,20 +38,16 @@ "name": "objectName", "widget-attributes": { "values": [ + "Bounce Event", "Data Extension", - "Campaign", "Email", - "Mailing List" + "Mailing List", + "Notsent Event", + "Open Event", + "Sent Event", + "Unsub Event" ], "default": "Data Extension" - } - }, - { - "widget-type": "textbox", - "label": "Data Extension External Key", - "name": "dataExtensionKey", - "widget-attributes": { - "placeholder": "External key of the Data Extension from which data to be read." }, "plugin-function": { "label": "Get Schema", @@ -63,6 +59,14 @@ } ] } + }, + { + "widget-type": "textbox", + "label": "Data Extension External Key", + "name": "dataExtensionKey", + "widget-attributes": { + "placeholder": "External key of the Data Extension from which data to be read." + } } ] }, @@ -76,12 +80,12 @@ "widget-attributes": { "options": [ { - "id": "Data Extension", - "label": "Data Extension" + "id": "Bounce Event", + "label": "Bounce Event" }, { - "id": "Campaign", - "label": "Campaign" + "id": "Data Extension", + "label": "Data Extension" }, { "id": "Email", @@ -90,6 +94,22 @@ { "id": "Mailing List", "label": "Mailing List" + }, + { + "id": "Notsent Event", + "label": "Notsent Event" + }, + { + "id": "Open Event", + "label": "Open Event" + }, + { + "id": "Sent Event", + "label": "Sent Event" + }, + { + "id": "Unsub Event", + "label": "Unsub Event" } ], "delimiter": ",", @@ -115,6 +135,19 @@ } ] }, + { + "label": "Filter", + "properties": [ + { + "widget-type": "textbox", + "label": "filter", + "name": "filter", + "widget-attributes": { + "placeholder": "data filter criteria" + } + } + ] + }, { "label": "Authentication", "properties": [ From 9eecb680efa83317dc9309e9f3e7ba0ebdffc6aa Mon Sep 17 00:00:00 2001 From: Shivakumar Gokaram Date: Sun, 28 Feb 2021 15:49:36 -0500 Subject: [PATCH 17/19] Change from Colgate Testing --- docs/MarketingCloud-batchsource.md | 28 +++-- ...alesforce_source_multiobject_pipeline.json | 101 ----------------- ...lesforce_source_singleobject_pipeline.json | 102 ------------------ ...ltiobject_pipeline-cdap-data-pipeline.json | 101 +++++++++++++++++ ...enevent_withfilter-cdap-data-pipeline.json | 94 ++++++++++++++++ .../fuelsdk/PaginationETSoapObject.java | 1 - .../source/MarketingCloudRecordReader.java | 18 ++++ .../sfmc/source/MarketingCloudSource.java | 1 - .../sfmc/source/util/SchemaBuilder.java | 3 +- 9 files changed, 237 insertions(+), 212 deletions(-) delete mode 100644 examples/example_salesforce_source_multiobject_pipeline.json delete mode 100644 examples/example_salesforce_source_singleobject_pipeline.json create mode 100644 examples/example_sfmc_multiobject_pipeline-cdap-data-pipeline.json create mode 100644 examples/example_sfmc_openevent_withfilter-cdap-data-pipeline.json diff --git a/docs/MarketingCloud-batchsource.md b/docs/MarketingCloud-batchsource.md index a382ea8..654f737 100644 --- a/docs/MarketingCloud-batchsource.md +++ b/docs/MarketingCloud-batchsource.md @@ -25,11 +25,19 @@ Properties `Data Extension` - will allow user to fetch data for a single Data Extension object, -`Campaign` - will allow user to fetch data for Campaign object, - `Email` - will allow user to fetch data for Email object, -`Mailing List` - will allow user to fetch data for Mailing List object. +`Mailing List` - will allow user to fetch data for Mailing List object, + +`Bounce Event` - will allow user to fetch Tracking Bounce Events, + +`Open Event` - will allow user to fetch Tracking Open Events, + +`Unsub Event` - will allow user to fetch Tracking UnSubscribe Events, + +`Sent Event` - will allow user to fetch Tracking Sent Events, + +`UnSent Event` - will allow user to fetch Tracking UnSent Events. Note, this value will be ignored if the Mode is set to `Multi Object`. @@ -44,11 +52,19 @@ be ignored in following two cases: `Data Extension` - will allow user to fetch data for a single Data Extension object, -`Campaign` - will allow user to fetch data for Campaign object, - `Email` - will allow user to fetch data for Email object, -`Mailing List` - will allow user to fetch data for Mailing List object. +`Mailing List` - will allow user to fetch data for Mailing List object + +`Bounce Event` - will allow user to fetch Tracking Bounce Events, + +`Open Event` - will allow user to fetch Tracking Open Events, + +`Unsub Event` - will allow user to fetch Tracking UnSubscribe Events, + +`Sent Event` - will allow user to fetch Tracking Sent Events, + +`UnSent Event` - will allow user to fetch Tracking UnSent Events. Note, this value will be ignored if the Mode is set to `Single Object`. diff --git a/examples/example_salesforce_source_multiobject_pipeline.json b/examples/example_salesforce_source_multiobject_pipeline.json deleted file mode 100644 index b15ae06..0000000 --- a/examples/example_salesforce_source_multiobject_pipeline.json +++ /dev/null @@ -1,101 +0,0 @@ -{ - "artifact": { - "name": "cdap-data-pipeline", - "version": "6.1.1", - "scope": "SYSTEM", - "label": "Data Pipeline - Batch" - }, - "description": "", - "name": "Test_Salesforce_Source_MultiObject_Mode", - "config": { - "resources": { - "memoryMB": 2048, - "virtualCores": 1 - }, - "driverResources": { - "memoryMB": 2048, - "virtualCores": 1 - }, - "connections": [ - { - "from": "Salesforce Marketing", - "to": "GCS Multi File" - } - ], - "comments": [], - "postActions": [], - "properties": {}, - "processTimingEnabled": true, - "stageLoggingEnabled": false, - "stages": [ - { - "name": "Salesforce Marketing", - "plugin": { - "name": "Salesforce", - "type": "batchsource", - "label": "Salesforce Marketing", - "artifact": { - "name": "salesforce-marketing-plugins", - "version": "1.2.0-SNAPSHOT", - "scope": "USER" - }, - "properties": { - "queryMode": "Multi Object", - "tableNameField": "tablename", - "referenceName": "example_sf_sf_source_1", - "clientId": "${CLIENT_ID}", - "clientSecret": "${CLIENT_SECRET}", - "authEndpoint": "${AUTH_ENDPOINT}", - "soapEndpoint": "${SOAP_ENDPOINT}", - "restEndpoint": "${REST_ENDPOINT}", - "dataExtensionKeyList": "${DATA_EXTENSION_KEY_LIST}", - "objectName": "Data Extension", - "objectList": "${SFMC_OBJECT_LIST}", - "failOnError": "false" - } - }, - "outputSchema": [ - { - "name": "etlSchemaBody", - "schema": "" - } - ] - }, - { - "name": "GCS Multi File", - "plugin": { - "name": "GCSMultiFiles", - "type": "batchsink", - "label": "GCS Multi File", - "artifact": { - "name": "google-cloud", - "version": "0.13.2", - "scope": "SYSTEM" - }, - "properties": { - "project": "${GCS_PROJECT_ID}", - "suffix": "yyyy-MM-dd-HH-mm", - "format": "csv", - "serviceFilePath": "${GCS_SERVICE_ACCOUNT_FILE_PATH}", - "compressionCodec": "None", - "splitField": "tablename", - "location": "us", - "referenceName": "example_sf_gcs_multi_sink_1", - "path": "${GCS_BUCKET_PATH}", - "delimiter": "," - } - }, - "inputSchema": [ - { - "name": "Salesforce Marketing", - "schema": "" - } - ] - } - ], - "schedule": "0 * * * *", - "engine": "spark", - "numOfRecordsPreview": 100, - "maxConcurrentRuns": 1 - } -} \ No newline at end of file diff --git a/examples/example_salesforce_source_singleobject_pipeline.json b/examples/example_salesforce_source_singleobject_pipeline.json deleted file mode 100644 index 522546c..0000000 --- a/examples/example_salesforce_source_singleobject_pipeline.json +++ /dev/null @@ -1,102 +0,0 @@ -{ - "artifact": { - "name": "cdap-data-pipeline", - "version": "6.1.1", - "scope": "SYSTEM", - "label": "Data Pipeline - Batch" - }, - "description": "", - "name": "Test_Salesforce_Source_SingleObject_Mode", - "config": { - "resources": { - "memoryMB": 2048, - "virtualCores": 1 - }, - "driverResources": { - "memoryMB": 2048, - "virtualCores": 1 - }, - "connections": [ - { - "from": "Salesforce Marketing", - "to": "File" - } - ], - "comments": [], - "postActions": [], - "properties": {}, - "processTimingEnabled": true, - "stageLoggingEnabled": false, - "stages": [ - { - "name": "Salesforce Marketing", - "plugin": { - "name": "Salesforce", - "type": "batchsource", - "label": "Salesforce Marketing", - "artifact": { - "name": "salesforce-marketing-plugins", - "version": "1.2.0-SNAPSHOT", - "scope": "USER" - }, - "properties": { - "queryMode": "Single Object", - "tableNameField": "tablename", - "referenceName": "example_sf_sf_source_2", - "dataExtensionKey": "${DATA_EXTENSION_KEY}", - "clientId": "${CLIENT_ID}", - "clientSecret": "${CLIENT_SECRET}", - "authEndpoint": "${AUTH_ENDPOINT}", - "soapEndpoint": "${SOAP_ENDPOINT}", - "restEndpoint": "${REST_ENDPOINT}", - "objectName": "${SFMC_OBJECT_NAME}", - "objectList": "Data Extension", - "failOnError": "false" - } - }, - "outputSchema": [ - { - "name": "etlSchemaBody", - "schema": "" - } - ] - }, - { - "name": "File", - "plugin": { - "name": "File", - "type": "batchsink", - "label": "File", - "artifact": { - "name": "core-plugins", - "version": "2.3.4", - "scope": "SYSTEM" - }, - "properties": { - "suffix": "${OUTPUT_PATH_SUFFIX}", - "format": "csv", - "referenceName": "example_sf_file_sink_2", - "path": "${OUTPUT_FILE_PATH}", - "delimiter": "," - } - }, - "outputSchema": [ - { - "name": "etlSchemaBody", - "schema": "" - } - ], - "inputSchema": [ - { - "name": "Salesforce Marketing", - "schema": "" - } - ] - } - ], - "schedule": "0 * * * *", - "engine": "spark", - "numOfRecordsPreview": 100, - "maxConcurrentRuns": 1 - } -} \ No newline at end of file diff --git a/examples/example_sfmc_multiobject_pipeline-cdap-data-pipeline.json b/examples/example_sfmc_multiobject_pipeline-cdap-data-pipeline.json new file mode 100644 index 0000000..3c1f4a4 --- /dev/null +++ b/examples/example_sfmc_multiobject_pipeline-cdap-data-pipeline.json @@ -0,0 +1,101 @@ +{ + "artifact": { + "name": "cdap-data-pipeline", + "version": "6.2.1", + "scope": "SYSTEM" + }, + "description": "Data Pipeline Application", + "name": "example_sfmc_multiobject_pipeline", + "config": { + "resources": { + "memoryMB": 2048, + "virtualCores": 1 + }, + "driverResources": { + "memoryMB": 2048, + "virtualCores": 1 + }, + "connections": [ + { + "from": "Salesforce-Marketing", + "to": "File" + } + ], + "comments": [], + "postActions": [], + "properties": {}, + "processTimingEnabled": true, + "stageLoggingEnabled": false, + "stages": [ + { + "name": "Salesforce-Marketing", + "plugin": { + "name": "MarketingCloud", + "type": "batchsource", + "label": "Salesforce Marketing", + "artifact": { + "name": "salesforce-marketing-plugins", + "version": "1.2.2-SNAPSHOT", + "scope": "USER" + }, + "properties": { + "queryMode": "Multi Object", + "objectList": "Data Extension", + "tableNameField": "tablename", + "failOnError": "false", + "referenceName": "SFMC", + "clientId": "${SFMC_CLIENT_ID}", + "clientSecret": "${SFMC_CLIENT_SECRET}", + "authEndpoint": "${SFMC_AUTH_URI}", + "soapEndpoint": "${SFMC_SOAP_URI}", + "restEndpoint": "${SFMC_REST_URI}", + "dataExtensionKeyList": "CA397115-F3E1-4180-B7D0-AF91A60E8B2D,CFA3EE8B-320E-4E27-BBCC-CF4291556BE0" + } + }, + "outputSchema": [ + { + "name": "etlSchemaBody", + "schema": "" + } + ] + }, + { + "name": "File", + "plugin": { + "name": "File", + "type": "batchsink", + "label": "File", + "artifact": { + "name": "core-plugins", + "version": "2.4.1", + "scope": "SYSTEM" + }, + "properties": { + "suffix": "yyyy-MM-dd-HH-mm", + "format": "csv", + "referenceName": "SFMCData", + "path": "${OUTPUT_DIR}", + "schema": "{\"type\":\"record\",\"name\":\"etlSchemaBody\",\"fields\":[{\"name\":\"subscriber_key\",\"type\":[\"string\",\"null\"]},{\"name\":\"campaign_c\",\"type\":[\"string\",\"null\"]},{\"name\":\"email_address\",\"type\":[\"string\",\"null\"]},{\"name\":\"campaign_a\",\"type\":[\"string\",\"null\"]},{\"name\":\"account_unsub\",\"type\":[\"string\",\"null\"]},{\"name\":\"campaign_b\",\"type\":[\"string\",\"null\"]},{\"name\":\"first__name\",\"type\":[\"string\",\"null\"]},{\"name\":\"emailaddress\",\"type\":[\"string\",\"null\"]},{\"name\":\"subscriberkey\",\"type\":[\"string\",\"null\"]},{\"name\":\"tablename\",\"type\":\"string\"}]}" + } + }, + "outputSchema": [ + { + "name": "etlSchemaBody", + "schema": "{\"type\":\"record\",\"name\":\"etlSchemaBody\",\"fields\":[{\"name\":\"subscriber_key\",\"type\":[\"string\",\"null\"]},{\"name\":\"campaign_c\",\"type\":[\"string\",\"null\"]},{\"name\":\"email_address\",\"type\":[\"string\",\"null\"]},{\"name\":\"campaign_a\",\"type\":[\"string\",\"null\"]},{\"name\":\"account_unsub\",\"type\":[\"string\",\"null\"]},{\"name\":\"campaign_b\",\"type\":[\"string\",\"null\"]},{\"name\":\"first__name\",\"type\":[\"string\",\"null\"]},{\"name\":\"emailaddress\",\"type\":[\"string\",\"null\"]},{\"name\":\"subscriberkey\",\"type\":[\"string\",\"null\"]},{\"name\":\"tablename\",\"type\":\"string\"}]}" + } + ], + "inputSchema": [ + { + "name": "Salesforce Marketing", + "schema": "" + } + ] + } + ], + "schedule": "0 * * * *", + "engine": "spark", + "numOfRecordsPreview": 100, + "description": "Data Pipeline Application", + "maxConcurrentRuns": 1 + } +} \ No newline at end of file diff --git a/examples/example_sfmc_openevent_withfilter-cdap-data-pipeline.json b/examples/example_sfmc_openevent_withfilter-cdap-data-pipeline.json new file mode 100644 index 0000000..05471cf --- /dev/null +++ b/examples/example_sfmc_openevent_withfilter-cdap-data-pipeline.json @@ -0,0 +1,94 @@ +{ + "artifact": { + "name": "cdap-data-pipeline", + "version": "6.2.1", + "scope": "SYSTEM" + }, + "description": "", + "name": "example_sfmc_openevent_withfilter", + "config": { + "resources": { + "memoryMB": 2048, + "virtualCores": 1 + }, + "driverResources": { + "memoryMB": 2048, + "virtualCores": 1 + }, + "connections": [ + { + "from": "Salesforce-Marketing", + "to": "File" + } + ], + "comments": [], + "postActions": [], + "properties": {}, + "processTimingEnabled": true, + "stageLoggingEnabled": false, + "stages": [ + { + "name": "Salesforce-Marketing", + "plugin": { + "name": "MarketingCloud", + "type": "batchsource", + "label": "Salesforce Marketing", + "artifact": { + "name": "salesforce-marketing-plugins", + "version": "1.2.2-SNAPSHOT", + "scope": "USER" + }, + "properties": { + "queryMode": "Single Object", + "objectName": "Open Event", + "failOnError": "false", + "referenceName": "SFMC", + "clientId": "${SFMC_CLIENT_ID}", + "clientSecret": "${SFMC_CLIENT_SECRET}", + "authEndpoint": "${SFMC_AUTH_URI}", + "soapEndpoint": "${SFMC_SOAP_URI}", + "restEndpoint": "${SFMC_REST_URI}", + "filter": "subscriberKey = 'manoj.muralidharan@verticurl.com' and eventDate > '04-19-2019'" + } + }, + "outputSchema": [ + { + "name": "etlSchemaBody", + "schema": "{\"type\":\"record\",\"name\":\"etlSchemaBody\",\"fields\":[{\"name\":\"id\",\"type\":[\"string\",\"null\"]},{\"name\":\"sendID\",\"type\":[\"int\",\"null\"]},{\"name\":\"subscriberKey\",\"type\":[\"string\",\"null\"]},{\"name\":\"eventDate\",\"type\":[\"string\",\"null\"]},{\"name\":\"eventType\",\"type\":[\"string\",\"null\"]},{\"name\":\"triggeredSendDefinitionObjectID\",\"type\":[\"string\",\"null\"]},{\"name\":\"batchID\",\"type\":[\"int\",\"null\"]}]}" + } + ] + }, + { + "name": "File", + "plugin": { + "name": "File", + "type": "batchsink", + "label": "File", + "artifact": { + "name": "core-plugins", + "version": "2.4.1", + "scope": "SYSTEM" + }, + "properties": { + "suffix": "yyyy-MM-dd-HH-mm", + "format": "csv", + "schema": "{\"type\":\"record\",\"name\":\"etlSchemaBody\",\"fields\":[{\"name\":\"id\",\"type\":[\"string\",\"null\"]},{\"name\":\"sendID\",\"type\":[\"int\",\"null\"]},{\"name\":\"subscriberKey\",\"type\":[\"string\",\"null\"]},{\"name\":\"eventDate\",\"type\":[\"string\",\"null\"]},{\"name\":\"eventType\",\"type\":[\"string\",\"null\"]},{\"name\":\"triggeredSendDefinitionObjectID\",\"type\":[\"string\",\"null\"]},{\"name\":\"batchID\",\"type\":[\"int\",\"null\"]}]}", + "referenceName": "FILEOUT", + "path": "${OUTPUT_DIR}" + } + }, + "outputSchema": "{\"type\":\"record\",\"name\":\"etlSchemaBody\",\"fields\":[{\"name\":\"id\",\"type\":[\"string\",\"null\"]},{\"name\":\"sendID\",\"type\":[\"int\",\"null\"]},{\"name\":\"subscriberKey\",\"type\":[\"string\",\"null\"]},{\"name\":\"eventDate\",\"type\":[\"string\",\"null\"]},{\"name\":\"eventType\",\"type\":[\"string\",\"null\"]},{\"name\":\"triggeredSendDefinitionObjectID\",\"type\":[\"string\",\"null\"]},{\"name\":\"batchID\",\"type\":[\"int\",\"null\"]}]}", + "inputSchema": [ + { + "name": "Salesforce Marketing", + "schema": "{\"type\":\"record\",\"name\":\"etlSchemaBody\",\"fields\":[{\"name\":\"id\",\"type\":[\"string\",\"null\"]},{\"name\":\"sendID\",\"type\":[\"int\",\"null\"]},{\"name\":\"subscriberKey\",\"type\":[\"string\",\"null\"]},{\"name\":\"eventDate\",\"type\":[\"string\",\"null\"]},{\"name\":\"eventType\",\"type\":[\"string\",\"null\"]},{\"name\":\"triggeredSendDefinitionObjectID\",\"type\":[\"string\",\"null\"]},{\"name\":\"batchID\",\"type\":[\"int\",\"null\"]}]}" + } + ] + } + ], + "schedule": "0 * * * *", + "engine": "spark", + "numOfRecordsPreview": 100, + "maxConcurrentRuns": 1 + } +} \ No newline at end of file diff --git a/src/main/java/com/custom/fuelsdk/PaginationETSoapObject.java b/src/main/java/com/custom/fuelsdk/PaginationETSoapObject.java index 5741260..1f4b339 100644 --- a/src/main/java/com/custom/fuelsdk/PaginationETSoapObject.java +++ b/src/main/java/com/custom/fuelsdk/PaginationETSoapObject.java @@ -106,7 +106,6 @@ public static ETResponse continueRequest(ETClient client, St ETFilter filter) throws ETSdkException { ETResponse response = customRetrieve(client, soapObject, filter, requestID, ETDataExtensionRow.class); - logger.debug("final response: " + response); return response; } diff --git a/src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudRecordReader.java b/src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudRecordReader.java index 365b8fa..d1e071d 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudRecordReader.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudRecordReader.java @@ -36,8 +36,10 @@ import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Collections; +import java.util.Date; import java.util.Iterator; import java.util.List; +import java.util.concurrent.TimeUnit; /** * Record reader that reads the entire contents of a Salesforce table. @@ -245,6 +247,10 @@ private String createGetterName(String name) { */ private Object convertToValue(String fieldName, Schema fieldSchema, Object fieldValue) { Schema.Type fieldType = fieldSchema.getType(); + Schema.LogicalType logicalType = fieldSchema.getLogicalType(); + if (fieldSchema.getLogicalType() != null) { + return transformLogicalType(fieldName, logicalType, fieldValue); + } switch (fieldType) { case STRING: @@ -295,4 +301,16 @@ private Boolean convertToBooleanValue(Object fieldValue) { return Boolean.parseBoolean(String.valueOf(fieldValue)); } + private Object transformLogicalType(String fieldName, Schema.LogicalType logicalType, Object value) { + switch (logicalType) { + case TIMESTAMP_MICROS: + if (value instanceof Date) { + return TimeUnit.MILLISECONDS.toMicros((((Date) value).getTime())); + } + return null; + default: + throw new IllegalArgumentException( + String.format("Field '%s' is of unsupported type '%s'", fieldName, logicalType.getToken())); + } + } } diff --git a/src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudSource.java b/src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudSource.java index 9f0b5ec..8331048 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudSource.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudSource.java @@ -70,7 +70,6 @@ public MarketingCloudSource(MarketingCloudSourceConfig conf) { public void configurePipeline(PipelineConfigurer pipelineConfigurer) { super.configurePipeline(pipelineConfigurer); - LOG.debug("Validate config during `configurePipeline` stage: {}", conf); StageConfigurer stageConfigurer = pipelineConfigurer.getStageConfigurer(); FailureCollector collector = stageConfigurer.getFailureCollector(); diff --git a/src/main/java/io/cdap/plugin/sfmc/source/util/SchemaBuilder.java b/src/main/java/io/cdap/plugin/sfmc/source/util/SchemaBuilder.java index f2ac91e..cd68890 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/util/SchemaBuilder.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/util/SchemaBuilder.java @@ -74,8 +74,9 @@ private Schema createSchema(MarketingCloudColumn column) { return Schema.of(Schema.Type.INT); case "boolean": return Schema.of(Schema.Type.BOOLEAN); - case "phone": case "date": + return Schema.of(Schema.LogicalType.TIMESTAMP_MICROS); + case "phone": case "email_address": case "locale": case "string": From b1c33b98f7acda9d42a0995552596cd190468d4d Mon Sep 17 00:00:00 2001 From: Shivakumar Gokaram Date: Tue, 2 Mar 2021 00:24:13 -0500 Subject: [PATCH 18/19] Clean up of Code --- .../io/cdap/plugin/sfmc/source/MarketingCloudSource.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudSource.java b/src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudSource.java index 8331048..85f2352 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudSource.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudSource.java @@ -122,8 +122,6 @@ public void transform(KeyValue input, Emitter tables = MarketingCloudInputFormat.setInput(hConf, mode, conf); @@ -131,7 +129,6 @@ private Schema getSchema(SourceQueryMode mode) { schema = tables.iterator().next().getSchema(); } } - return schema; } @@ -144,7 +141,7 @@ private void recordLineage(BatchSourceContext context, MarketingCloudObjectInfo List fields = Objects.requireNonNull(schema).getFields(); if (fields != null && !fields.isEmpty()) { lineageRecorder.recordRead("Read", - String.format("Read from '%s' Salesforce object.", tableName), + String.format("Read from '%s' Marketing Cloud object.", tableName), fields.stream().map(Schema.Field::getName).collect(Collectors.toList())); } } From 40d80a00a2811c16edda286b5c19b68edbc421c6 Mon Sep 17 00:00:00 2001 From: Shivakumar Gokaram Date: Tue, 9 Mar 2021 22:28:13 -0500 Subject: [PATCH 19/19] Update for multi tables --- .../cdap/plugin/sfmc/source/MarketingCloudRecordReader.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudRecordReader.java b/src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudRecordReader.java index d1e071d..2b73105 100644 --- a/src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudRecordReader.java +++ b/src/main/java/io/cdap/plugin/sfmc/source/MarketingCloudRecordReader.java @@ -56,6 +56,7 @@ public class MarketingCloudRecordReader extends RecordReader results; private Iterator iterator; @@ -106,7 +107,7 @@ public StructuredRecord getCurrentValue() throws IOException { StructuredRecord.Builder recordBuilder = StructuredRecord.builder(schema); if (pluginConf.getQueryMode() == SourceQueryMode.MULTI_OBJECT) { - recordBuilder.set(tableNameField, tableName); + recordBuilder.set(tableNameField, formattedTableName); } try { @@ -136,6 +137,8 @@ public void close() throws IOException { private void fetchData() { object = SourceObject.valueOf(split.getObjectName()); tableName = split.getTableName(); + formattedTableName = tableName.replaceAll("-", "_"); + if (object == SourceObject.DATA_EXTENSION) { dataExtensionKey = tableName.replaceAll(MarketingCloudConstants.DATA_EXTENSION_PREFIX, ""); }