From 7a763cf54b2fd2540b7c5d8ddd25238019069f96 Mon Sep 17 00:00:00 2001 From: Yevhenii Chekanskyi Date: Thu, 31 Oct 2019 13:39:16 +0200 Subject: [PATCH 01/12] PLUGIN-75 Marketo Plugin - initial skeleton for entity plugin. --- .gitignore | 3 + checkstyle.xml | 406 ++++++++++++++++++ pom.xml | 297 +++++++++++++ .../cdap/plugin/marketo/common/Marketo.java | 151 +++++++ .../plugin/marketo/common/MarketoEntity.java | 56 +++ .../marketo/common/MarketoSchemaReader.java | 60 +++ .../plugin/marketo/common/MarketoToken.java | 55 +++ .../marketo/common/response/MarketoPage.java | 80 ++++ .../response/describe/DescribeResponse.java | 45 ++ .../describe/DescribeResponseLeads.java | 35 ++ .../common/response/describe/Field.java | 51 +++ .../source/batch/MarketoBatchSource.java | 90 ++++ .../batch/MarketoBatchSourceConfig.java | 153 +++++++ .../source/batch/MarketoInputFormat.java | 41 ++ .../batch/MarketoInputFormatProvider.java | 51 +++ .../source/batch/MarketoRecordReader.java | 75 ++++ .../source/batch/NoOpMarketoSplit.java | 49 +++ suppressions.xml | 33 ++ 18 files changed, 1731 insertions(+) create mode 100644 .gitignore create mode 100644 checkstyle.xml create mode 100644 pom.xml create mode 100644 src/main/java/io/cdap/plugin/marketo/common/Marketo.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/MarketoEntity.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/MarketoSchemaReader.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/MarketoToken.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/response/MarketoPage.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/response/describe/DescribeResponse.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/response/describe/DescribeResponseLeads.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/response/describe/Field.java create mode 100644 src/main/java/io/cdap/plugin/marketo/source/batch/MarketoBatchSource.java create mode 100644 src/main/java/io/cdap/plugin/marketo/source/batch/MarketoBatchSourceConfig.java create mode 100644 src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormat.java create mode 100644 src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormatProvider.java create mode 100644 src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java create mode 100644 src/main/java/io/cdap/plugin/marketo/source/batch/NoOpMarketoSplit.java create mode 100644 suppressions.xml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..34e1547 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +target +.idea +*.iml \ No newline at end of file diff --git a/checkstyle.xml b/checkstyle.xml new file mode 100644 index 0000000..fb35f2d --- /dev/null +++ b/checkstyle.xml @@ -0,0 +1,406 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..60f2184 --- /dev/null +++ b/pom.xml @@ -0,0 +1,297 @@ + + + 4.0.0 + + io.cdap + marketo-entity-plugin + 1.0-SNAPSHOT + + + + sonatype + https://oss.sonatype.org/content/groups/public + + + sonatype-snapshots + https://oss.sonatype.org/content/repositories/snapshots + + + + + 6.1.0-SNAPSHOT + 2.3.0 + 2.3.0-SNAPSHOT + 2.1.3 + + + + + io.cdap.cdap + cdap-api + ${cdap.version} + provided + + + io.cdap.cdap + cdap-etl-api + ${cdap.version} + provided + + + io.cdap.cdap + cdap-formats + ${cdap.version} + + + org.apache.avro + avro + + + io.thekraken + grok + + + + + io.cdap.plugin + hydrator-common + ${hydrator.version} + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + provided + + + commons-logging + commons-logging + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + org.apache.avro + avro + + + org.apache.zookeeper + zookeeper + + + com.google.guava + guava + + + jersey-core + com.sun.jersey + + + jersey-json + com.sun.jersey + + + jersey-server + com.sun.jersey + + + servlet-api + javax.servlet + + + org.mortbay.jetty + jetty + + + org.mortbay.jetty + jetty-util + + + jasper-compiler + tomcat + + + jasper-runtime + tomcat + + + jsp-api + javax.servlet.jsp + + + slf4j-api + org.slf4j + + + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + provided + + + org.slf4j + slf4j-log4j12 + + + com.google.inject.extensions + guice-servlet + + + com.sun.jersey + jersey-core + + + com.sun.jersey + jersey-server + + + com.sun.jersey + jersey-json + + + com.sun.jersey.contribs + jersey-guice + + + javax.servlet + servlet-api + + + com.google.guava + guava + + + + + + io.cdap.cdap + cdap-etl-api-spark + ${cdap.version} + provided + + + org.apache.spark + spark-streaming_2.11 + ${spark2.version} + provided + + + org.apache.spark + spark-core_2.11 + ${spark2.version} + provided + + + org.slf4j + slf4j-log4j12 + + + log4j + log4j + + + org.apache.hadoop + hadoop-client + + + com.esotericsoftware.reflectasm + reflectasm + + + org.apache.curator + curator-recipes + + + + org.scala-lang + scala-compiler + + + org.scala-lang + scala-reflect + + + org.eclipse.jetty.orbit + javax.servlet + + + + net.java.dev.jets3t + jets3t + + + asm + asm + + + + + + com.google.code.gson + gson + 2.8.6 + + + + junit + junit + 4.12 + test + + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + 2.17 + + + validate + process-test-classes + + checkstyle.xml + suppressions.xml + UTF-8 + true + true + true + **/org/apache/cassandra/**,**/org/apache/hadoop/** + + + check + + + + + + com.puppycrawl.tools + checkstyle + 6.19 + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + + \ No newline at end of file diff --git a/src/main/java/io/cdap/plugin/marketo/common/Marketo.java b/src/main/java/io/cdap/plugin/marketo/common/Marketo.java new file mode 100644 index 0000000..2c5fd18 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/Marketo.java @@ -0,0 +1,151 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common; + +import com.google.common.base.Strings; +import com.google.gson.Gson; +import io.cdap.plugin.marketo.common.response.MarketoPage; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.net.URL; +import java.util.Iterator; +import java.util.Map; +import java.util.NoSuchElementException; +import javax.net.ssl.HttpsURLConnection; + +/** + * Helper class to perform Marketo Api operations. + */ +public class Marketo { + /** + * Marketo page iterator. + */ + public static class MarketoPageIterator implements Iterator> { + private MarketoPage currentPage; + private Marketo marketo; + private String queryUrl; + private Iterator> currentPageResultIterator; + + private MarketoPageIterator(MarketoPage page, Marketo marketo, String queryUrl) { + this.currentPage = page; + this.marketo = marketo; + this.queryUrl = queryUrl; + currentPageResultIterator = currentPage.getResults().iterator(); + } + + @Override + public boolean hasNext() { + if (currentPageResultIterator.hasNext()) { + return true; + } else { + MarketoPage nextPage = marketo.getNextPage(currentPage, queryUrl); + if (nextPage != null) { + currentPage = nextPage; + currentPage.getResults().iterator(); + return hasNext(); + } else { + return false; + } + } + } + + @Override + public Map next() { + if (hasNext()) { + return currentPageResultIterator.next(); + } else { + throw new NoSuchElementException(); + } + } + } + + private static final Gson GSON = new Gson(); + + private String marketoEndpoint; + private MarketoToken token; + + public Marketo(String marketoEndpoint, String clientId, String clientSecret) { + this.marketoEndpoint = marketoEndpoint; + token = getToken(marketoEndpoint, clientId, clientSecret); + } + + public Marketo(String marketoEndpoint, MarketoToken token) { + this.marketoEndpoint = marketoEndpoint; + this.token = token; + } + + public MarketoPage getPage(String queryUrl) { + return get(queryUrl, MarketoPage.class); + } + + public MarketoPage getNextPage(MarketoPage page, String queryUrl) { + if (!Strings.isNullOrEmpty(page.getNextPageToken())) { + return getPage(queryUrl + "&nextPageToken=" + page.getNextPageToken()); + } + return null; + } + + public MarketoPageIterator iteratePage(String queryUrl) { + return new MarketoPageIterator(getPage(queryUrl), this, queryUrl); + } + + private String appendToken(String requestUrl) { + if (!requestUrl.contains("access_token")) { + if (requestUrl.contains("?")) { + return requestUrl + "&access_token=" + token.getAccessToken(); + } else { + return requestUrl + "?access_token=" + token.getAccessToken(); + } + } + return requestUrl; + } + + public T get(String queryUrl, Class cls) { + return doGet(appendToken(marketoEndpoint + queryUrl), cls); + } + + public static MarketoToken getToken(String marketoEndpoint, String clientId, String clientSecret) { + String marketoIdURL = marketoEndpoint + "/identity"; + String idEndpoint = marketoIdURL + "/oauth/token?grant_type=client_credentials&client_id=" + + clientId + "&client_secret=" + clientSecret; + + return doGet(idEndpoint, MarketoToken.class); + } + + + private static T doGet(String queryUrl, Class cls) { + try { + URL url = new URL(queryUrl); + HttpsURLConnection urlConn = (HttpsURLConnection) url.openConnection(); + urlConn.setRequestMethod("GET"); + urlConn.setRequestProperty("accept", "application/json"); + int responseCode = urlConn.getResponseCode(); + if (responseCode == 200) { + InputStream inStream = urlConn.getInputStream(); + Reader reader = new InputStreamReader(inStream); + return GSON.fromJson(reader, cls); + } else { + throw new IOException("Status: " + responseCode); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/MarketoEntity.java b/src/main/java/io/cdap/plugin/marketo/common/MarketoEntity.java new file mode 100644 index 0000000..111f239 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/MarketoEntity.java @@ -0,0 +1,56 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common; + +import java.util.Arrays; + +/** + * Marketo Entity types. + */ +public enum MarketoEntity { + Leads("Leads", "GET /rest/v1/leads.json", "/rest/v1/leads/describe.json"), + Campaigns("Campaigns", "/rest/v1/campaigns.json", null), + Companies("Companies", "/rest/v1/companies.json", "/rest/v1/companies/describe.json"); + + private String name; + private String getEndpoint; + private String describeEndpoint; + + MarketoEntity(String name, String getEndpoint, String describeEndpoint) { + this.name = name; + this.getEndpoint = getEndpoint; + this.describeEndpoint = describeEndpoint; + } + + public String getName() { + return name; + } + + public String getGetEndpoint() { + return getEndpoint; + } + + public String getDescribeEndpoint() { + return describeEndpoint; + } + + public static MarketoEntity fromString(String marketoEntity) { + return Arrays.stream(MarketoEntity.values()) + .filter(entity -> entity.getName().equals(marketoEntity)).findFirst() + .orElseThrow(() -> new RuntimeException(String.format("'%s' is not a valid Marketo entity", marketoEntity))); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/MarketoSchemaReader.java b/src/main/java/io/cdap/plugin/marketo/common/MarketoSchemaReader.java new file mode 100644 index 0000000..6d4996b --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/MarketoSchemaReader.java @@ -0,0 +1,60 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common; + +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.marketo.common.response.describe.DescribeResponse; +import io.cdap.plugin.marketo.common.response.describe.DescribeResponseLeads; +import io.cdap.plugin.marketo.common.response.describe.Field; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Generates schema for given entity. + */ +public class MarketoSchemaReader { + // TODO fill predefined schemas + private static final Map PREDEFINED_SCHEMAS = new HashMap<>(); + + public static Schema getSchemaForEntity(MarketoEntity entity, Marketo marketo) { + if (entity.getDescribeEndpoint() != null) { + List entityFields; + + if (entity == MarketoEntity.Leads) { + DescribeResponseLeads leadsDescribe = marketo.get(entity.getDescribeEndpoint(), + DescribeResponseLeads.class); + entityFields = leadsDescribe.getFields(); + } else { + DescribeResponse describe = marketo.get(entity.getDescribeEndpoint(), DescribeResponse.class); + entityFields = describe.getFields(); + } + + List schemaFields = entityFields.stream() + //TODO map types here + .map(field -> Schema.Field.of(field.getName(), Schema.nullableOf(Schema.of(Schema.Type.STRING)))) + .collect(Collectors.toList()); + return Schema.recordOf(entity.getName(), schemaFields); + } else if (PREDEFINED_SCHEMAS.containsKey(entity)) { + return PREDEFINED_SCHEMAS.get(entity); + } + throw new RuntimeException(String.format("Unable to get schema for entity '%s'", entity.getName())); + } + +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/MarketoToken.java b/src/main/java/io/cdap/plugin/marketo/common/MarketoToken.java new file mode 100644 index 0000000..6f7c235 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/MarketoToken.java @@ -0,0 +1,55 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common; + +import com.google.gson.annotations.SerializedName; + +/** + * Marketo Token holder. + */ +public class MarketoToken { + @SerializedName("access_token") + private String accessToken; + private String scope; + @SerializedName("expires_in") + private String expiresIn; + @SerializedName("token_type") + private String tokenType; + + public MarketoToken(String accessToken, String scope, String expiresIn, String tokenType) { + this.accessToken = accessToken; + this.scope = scope; + this.expiresIn = expiresIn; + this.tokenType = tokenType; + } + + public String getAccessToken() { + return accessToken; + } + + public String getScope() { + return scope; + } + + public String getExpiresIn() { + return expiresIn; + } + + public String getTokenType() { + return tokenType; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/response/MarketoPage.java b/src/main/java/io/cdap/plugin/marketo/common/response/MarketoPage.java new file mode 100644 index 0000000..5311e7f --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/response/MarketoPage.java @@ -0,0 +1,80 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.response; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * Marketo page response. + */ +public class MarketoPage { + /** + * Error or warn message description. + */ + public static class Description { + private String code; + private String message; + + public String getCode() { + return code; + } + + public String getMessage() { + return message; + } + } + + private Boolean moreResult = null; + private String nextPageToken = null; + private String requestId = null; + private Boolean success = null; + + private List errors = Collections.emptyList(); + private List warnings = Collections.emptyList(); + + private List> result = Collections.emptyList(); + + public Boolean hasMoreResults() { + return moreResult; + } + + public String getNextPageToken() { + return nextPageToken; + } + + public String getRequestId() { + return requestId; + } + + public Boolean isSuccess() { + return success; + } + + public List getErrors() { + return errors; + } + + public List getWarnings() { + return warnings; + } + + public List> getResults() { + return result; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/response/describe/DescribeResponse.java b/src/main/java/io/cdap/plugin/marketo/common/response/describe/DescribeResponse.java new file mode 100644 index 0000000..23cc88f --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/response/describe/DescribeResponse.java @@ -0,0 +1,45 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.response.describe; + +import java.util.Collections; +import java.util.List; + +/** + * Regular describe response. + * Fields located in first object in 'result' field. + */ +public class DescribeResponse { + /** + * Result holder. + */ + public static class Result { + List fields = Collections.emptyList(); + } + + private String requestId = null; + private Boolean success = null; + + private List result = Collections.emptyList(); + + public List getFields() { + if (result.size() != 1) { + throw new RuntimeException("Expected to have one result."); + } + return result.get(0).fields; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/response/describe/DescribeResponseLeads.java b/src/main/java/io/cdap/plugin/marketo/common/response/describe/DescribeResponseLeads.java new file mode 100644 index 0000000..e5eefea --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/response/describe/DescribeResponseLeads.java @@ -0,0 +1,35 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.response.describe; + +import java.util.Collections; +import java.util.List; + +/** + * Describe response for Leads-Leads entity. + * Fields here is directly located in 'result' field of response. + */ +public class DescribeResponseLeads { + private String requestId = null; + private Boolean success = null; + + List result = Collections.emptyList(); + + public List getFields() { + return result; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/response/describe/Field.java b/src/main/java/io/cdap/plugin/marketo/common/response/describe/Field.java new file mode 100644 index 0000000..7086298 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/response/describe/Field.java @@ -0,0 +1,51 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.response.describe; + +/** + * Marketo field description. + */ +public class Field { + /** + * Soap or rest descriptor. + */ + public static class FieldDescription { + String name; + Boolean readOnly; + } + + String id = null; + private String name = null; + String displayName = null; + String dataType = null; + int length = -1; + Boolean updateable = null; + FieldDescription rest = null; + FieldDescription soap = null; + + public String getName() { + if (name == null) { + if (rest == null) { + throw new RuntimeException("Failed to get name for field."); + } else { + return rest.name; + } + } else { + return name; + } + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoBatchSource.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoBatchSource.java new file mode 100644 index 0000000..ad6545e --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoBatchSource.java @@ -0,0 +1,90 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.source.batch; + +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.api.annotation.Plugin; +import io.cdap.cdap.api.data.batch.Input; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.api.dataset.lib.KeyValue; +import io.cdap.cdap.etl.api.Emitter; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.cdap.etl.api.PipelineConfigurer; +import io.cdap.cdap.etl.api.batch.BatchSource; +import io.cdap.cdap.etl.api.batch.BatchSourceContext; +import io.cdap.plugin.common.LineageRecorder; +import org.apache.hadoop.io.NullWritable; + +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * Plugin that reads entities from Marketo api. + */ +@Plugin(type = BatchSource.PLUGIN_TYPE) +@Name(MarketoBatchSource.NAME) +@Description("Reads entities from Marketo.") +public class MarketoBatchSource extends BatchSource, StructuredRecord> { + public static final String NAME = "MarketoEntityPlugin"; + + private final MarketoBatchSourceConfig config; + + public MarketoBatchSource(MarketoBatchSourceConfig config) { + this.config = config; + + } + + @Override + public void configurePipeline(PipelineConfigurer pipelineConfigurer) { + validateConfiguration(pipelineConfigurer.getStageConfigurer().getFailureCollector()); + pipelineConfigurer.getStageConfigurer().setOutputSchema(config.getSchema()); + } + + @Override + public void prepareRun(BatchSourceContext batchSourceContext) { + validateConfiguration(batchSourceContext.getFailureCollector()); + LineageRecorder lineageRecorder = new LineageRecorder(batchSourceContext, config.referenceName); + lineageRecorder.createExternalDataset(config.getSchema()); + lineageRecorder.recordRead("Read", "Reading Marketo entities", + Objects.requireNonNull(config.getSchema().getFields()).stream() + .map(Schema.Field::getName) + .collect(Collectors.toList())); + + batchSourceContext.setInput(Input.of(config.referenceName, new MarketoInputFormatProvider(config))); + } + + @Override + public void transform(KeyValue> input, Emitter emitter) { + StructuredRecord.Builder builder = StructuredRecord.builder(config.getSchema()); + Map inputMap = input.getValue(); + config.getSchema().getFields().forEach( + field -> { + if (inputMap.containsKey(field.getName())) { + //TODO map fields, also refactor io.cdap.plugin.marketo.common.MarketoSchemaReader.getSchemaForEntity + builder.set(field.getName(), String.valueOf(inputMap.remove(field.getName()))); + } + } + ); + } + + private void validateConfiguration(FailureCollector failureCollector) { + failureCollector.getOrThrowException(); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoBatchSourceConfig.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoBatchSourceConfig.java new file mode 100644 index 0000000..f7f16f2 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoBatchSourceConfig.java @@ -0,0 +1,153 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.source.batch; + +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Macro; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.common.ReferencePluginConfig; +import io.cdap.plugin.marketo.common.Marketo; +import io.cdap.plugin.marketo.common.MarketoEntity; +import io.cdap.plugin.marketo.common.MarketoSchemaReader; +import io.cdap.plugin.marketo.common.MarketoToken; + +/** + * Provides all required configuration for reading Marketo entities. + */ +public class MarketoBatchSourceConfig extends ReferencePluginConfig { + public static final String PROPERTY_ENTITY_NAME = "entityName"; + public static final String PROPERTY_CLIENT_ID = "clientId"; + public static final String PROPERTY_CLIENT_SECRET = "clientSecret"; + public static final String PROPERTY_REST_API_ENDPOINT = "restApiEndpoint"; + public static final String PROPERTY_REST_API_IDENTITY = "restApiIdentity"; + public static final String PROPERTY_DAILY_API_LIMIT = "dailyApiLimit"; + public static final String PROPERTY_REPORT_FORMAT = "reportFormat"; + public static final String PROPERTY_START_DATE = "startDate"; + public static final String PROPERTY_END_DATE = "endDate"; + + @Name(PROPERTY_ENTITY_NAME) + @Description("Marketo entity name to fetch.") + @Macro + protected String entityName; + + @Name(PROPERTY_CLIENT_ID) + @Description("Marketo Client ID.") + @Macro + protected String clientId; + + @Name(PROPERTY_CLIENT_SECRET) + @Description("Marketo Client secret.") + @Macro + protected String clientSecret; + + @Name(PROPERTY_REST_API_ENDPOINT) + @Description("REST API endpoint URL.") + @Macro + protected String restApiEndpoint; + + @Name(PROPERTY_REST_API_IDENTITY) + @Description("REST API identity.") + @Macro + protected String restApiIdentity; + + @Name(PROPERTY_DAILY_API_LIMIT) + @Description("Marketo enforced daily API limit.") + @Macro + protected String dailyApiLimit; + + @Name(PROPERTY_REPORT_FORMAT) + @Description("Report format.") + @Macro + protected String reportFormat; + + @Name(PROPERTY_START_DATE) + @Description("Start date for the report.") + @Macro + protected String startDate; + + @Name(PROPERTY_END_DATE) + @Description("End date for the report.") + @Macro + protected String endDate; + + + private transient MarketoToken token = null; + private transient Schema schema = null; + private transient Marketo marketo = null; + + public MarketoBatchSourceConfig(String referenceName) { + super(referenceName); + } + + public Schema getSchema() { + if (schema == null) { + schema = MarketoSchemaReader.getSchemaForEntity(getEntityType(), getMarketo()); + } + return schema; + } + + public Marketo getMarketo() { + if (marketo == null) { + marketo = new Marketo(getRestApiEndpoint(), getMarketoToken()); + } + return marketo; + } + + public MarketoEntity getEntityType() { + return MarketoEntity.fromString(entityName); + } + + public String getClientId() { + return clientId; + } + + public String getClientSecret() { + return clientSecret; + } + + public String getRestApiEndpoint() { + return restApiEndpoint; + } + + public String getRestApiIdentity() { + return restApiIdentity; + } + + public String getDailyApiLimit() { + return dailyApiLimit; + } + + public String getReportFormat() { + return reportFormat; + } + + public String getStartDate() { + return startDate; + } + + public String getEndDate() { + return endDate; + } + + public MarketoToken getMarketoToken() { + if (token == null) { + token = Marketo.getToken(getRestApiEndpoint(), getClientId(), getClientSecret()); + } + return token; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormat.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormat.java new file mode 100644 index 0000000..4fd514e --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormat.java @@ -0,0 +1,41 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.source.batch; + +import org.apache.hadoop.mapreduce.InputFormat; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; + +import java.util.Collections; +import java.util.List; + +/** + * InputFormat for mapreduce job, which provides a single split of data. + */ +public class MarketoInputFormat extends InputFormat { + @Override + public List getSplits(JobContext jobContext) { + return Collections.singletonList(new NoOpMarketoSplit()); + } + + @Override + public RecordReader createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) { + return new MarketoRecordReader(); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormatProvider.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormatProvider.java new file mode 100644 index 0000000..b1bebb7 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormatProvider.java @@ -0,0 +1,51 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.source.batch; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.cdap.cdap.api.data.batch.InputFormatProvider; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * InputFormatProvider used by cdap to provide configurations to mapreduce job + */ +public class MarketoInputFormatProvider implements InputFormatProvider { + public static final String PROPERTY_CONFIG_JSON = "cdap.marketo.entity.config"; + private static final Gson gson = new GsonBuilder().create(); + private final Map conf; + + + MarketoInputFormatProvider(MarketoBatchSourceConfig config) { + this.conf = Collections.unmodifiableMap(new HashMap() {{ + put(PROPERTY_CONFIG_JSON, gson.toJson(config)); + }}); + } + + @Override + public String getInputFormatClassName() { + return MarketoInputFormat.class.getName(); + } + + @Override + public Map getInputFormatConfiguration() { + return conf; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java new file mode 100644 index 0000000..9d36451 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java @@ -0,0 +1,75 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.source.batch; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.cdap.plugin.marketo.common.Marketo; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapreduce.InputSplit; +import org.apache.hadoop.mapreduce.RecordReader; +import org.apache.hadoop.mapreduce.TaskAttemptContext; + +import java.io.IOException; +import java.util.Map; + +/** + * RecordReader implementation, which reads events from Marketo api. + */ +public class MarketoRecordReader extends RecordReader> { + private static final Gson GSON = new GsonBuilder().create(); + private Marketo.MarketoPageIterator iterator; + private Map current = null; + + @Override + public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException { + Configuration conf = taskAttemptContext.getConfiguration(); + String configJson = conf.get(MarketoInputFormatProvider.PROPERTY_CONFIG_JSON); + MarketoBatchSourceConfig config = GSON.fromJson(configJson, MarketoBatchSourceConfig.class); + + iterator = config.getMarketo().iteratePage(config.getEntityType().getGetEndpoint()); + } + + @Override + public boolean nextKeyValue() { + if (iterator.hasNext()) { + current = iterator.next(); + return true; + } + return false; + } + + @Override + public NullWritable getCurrentKey() { + return null; + } + + @Override + public Map getCurrentValue() { + return current; + } + + @Override + public float getProgress() { + return 0; + } + + @Override + public void close() { + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/NoOpMarketoSplit.java b/src/main/java/io/cdap/plugin/marketo/source/batch/NoOpMarketoSplit.java new file mode 100644 index 0000000..396ad97 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/NoOpMarketoSplit.java @@ -0,0 +1,49 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.source.batch; + +import org.apache.hadoop.io.Writable; +import org.apache.hadoop.mapreduce.InputSplit; + +import java.io.DataInput; +import java.io.DataOutput; + +/** + * A no-op split. + */ +public class NoOpMarketoSplit extends InputSplit implements Writable { + public NoOpMarketoSplit() { + } + + @Override + public void readFields(DataInput dataInput) { + } + + @Override + public void write(DataOutput dataOutput) { + } + + @Override + public long getLength() { + return 0; + } + + @Override + public String[] getLocations() { + return new String[0]; + } +} diff --git a/suppressions.xml b/suppressions.xml new file mode 100644 index 0000000..600350b --- /dev/null +++ b/suppressions.xml @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + From 1cfa51feab80417d7296c4c4a31a74bd6b1f57de Mon Sep 17 00:00:00 2001 From: Yevhenii Chekanskyi Date: Mon, 25 Nov 2019 14:09:36 +0200 Subject: [PATCH 02/12] PLUGIN-75 Marketo Plugin - bulk export of leads. --- pom.xml | 51 ++++++ .../cdap/plugin/marketo/common/Marketo.java | 151 ------------------ .../plugin/marketo/common/MarketoEntity.java | 56 ------- .../marketo/common/MarketoSchemaReader.java | 60 ------- .../plugin/marketo/common/api/HttpHelper.java | 106 ++++++++++++ .../marketo/common/api/LeadsExportJob.java | 63 ++++++++ .../plugin/marketo/common/api/Marketo.java | 121 ++++++++++++++ .../common/api/MarketoPageIterator.java | 59 +++++++ .../cdap/plugin/marketo/common/api/Urls.java | 12 ++ .../common/api/entities/BaseResponse.java | 41 +++++ .../marketo/common/api/entities/Error.java | 22 +++ .../{ => api/entities}/MarketoToken.java | 4 +- .../marketo/common/api/entities/Warning.java | 22 +++ .../api/entities/leads/LeadsDescribe.java | 69 ++++++++ .../api/entities/leads/LeadsExport.java | 82 ++++++++++ .../entities/leads/LeadsExportRequest.java | 107 +++++++++++++ .../marketo/common/response/MarketoPage.java | 80 ---------- .../response/describe/DescribeResponse.java | 45 ------ .../describe/DescribeResponseLeads.java | 35 ---- .../common/response/describe/Field.java | 51 ------ .../batch/MarketoInputFormatProvider.java | 4 +- .../source/batch/MarketoRecordReader.java | 50 +++++- ...ource.java => MarketoReportingPlugin.java} | 16 +- ...java => MarketoReportingSourceConfig.java} | 29 ++-- .../MarketoReportingPlugin-batchsource.json | 28 ++++ 25 files changed, 856 insertions(+), 508 deletions(-) delete mode 100644 src/main/java/io/cdap/plugin/marketo/common/Marketo.java delete mode 100644 src/main/java/io/cdap/plugin/marketo/common/MarketoEntity.java delete mode 100644 src/main/java/io/cdap/plugin/marketo/common/MarketoSchemaReader.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/HttpHelper.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/MarketoPageIterator.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/Urls.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/BaseResponse.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/Error.java rename src/main/java/io/cdap/plugin/marketo/common/{ => api/entities}/MarketoToken.java (93%) create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/Warning.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsDescribe.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExport.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportRequest.java delete mode 100644 src/main/java/io/cdap/plugin/marketo/common/response/MarketoPage.java delete mode 100644 src/main/java/io/cdap/plugin/marketo/common/response/describe/DescribeResponse.java delete mode 100644 src/main/java/io/cdap/plugin/marketo/common/response/describe/DescribeResponseLeads.java delete mode 100644 src/main/java/io/cdap/plugin/marketo/common/response/describe/Field.java rename src/main/java/io/cdap/plugin/marketo/source/batch/{MarketoBatchSource.java => MarketoReportingPlugin.java} (83%) rename src/main/java/io/cdap/plugin/marketo/source/batch/{MarketoBatchSourceConfig.java => MarketoReportingSourceConfig.java} (82%) create mode 100644 widgets/MarketoReportingPlugin-batchsource.json diff --git a/pom.xml b/pom.xml index 60f2184..fb9a529 100644 --- a/pom.xml +++ b/pom.xml @@ -243,6 +243,16 @@ gson 2.8.6 + + org.apache.commons + commons-csv + 1.7 + + + commons-io + commons-io + 2.6 + junit @@ -292,6 +302,47 @@ 8 + + io.cdap + cdap-maven-plugin + 1.1.0 + + + system:cdap-data-pipeline[6.1.0-SNAPSHOT,7.0.0-SNAPSHOT) + + + + + create-artifact-config + prepare-package + + create-plugin-json + + + + + + org.apache.felix + maven-bundle-plugin + 3.5.0 + true + + + <_exportcontents>io.cdap.plugin.marketo.* + *;inline=false;scope=compile + true + lib + + + + + package + + bundle + + + + \ No newline at end of file diff --git a/src/main/java/io/cdap/plugin/marketo/common/Marketo.java b/src/main/java/io/cdap/plugin/marketo/common/Marketo.java deleted file mode 100644 index 2c5fd18..0000000 --- a/src/main/java/io/cdap/plugin/marketo/common/Marketo.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright © 2019 Cask Data, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package io.cdap.plugin.marketo.common; - -import com.google.common.base.Strings; -import com.google.gson.Gson; -import io.cdap.plugin.marketo.common.response.MarketoPage; - -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.Reader; -import java.net.URL; -import java.util.Iterator; -import java.util.Map; -import java.util.NoSuchElementException; -import javax.net.ssl.HttpsURLConnection; - -/** - * Helper class to perform Marketo Api operations. - */ -public class Marketo { - /** - * Marketo page iterator. - */ - public static class MarketoPageIterator implements Iterator> { - private MarketoPage currentPage; - private Marketo marketo; - private String queryUrl; - private Iterator> currentPageResultIterator; - - private MarketoPageIterator(MarketoPage page, Marketo marketo, String queryUrl) { - this.currentPage = page; - this.marketo = marketo; - this.queryUrl = queryUrl; - currentPageResultIterator = currentPage.getResults().iterator(); - } - - @Override - public boolean hasNext() { - if (currentPageResultIterator.hasNext()) { - return true; - } else { - MarketoPage nextPage = marketo.getNextPage(currentPage, queryUrl); - if (nextPage != null) { - currentPage = nextPage; - currentPage.getResults().iterator(); - return hasNext(); - } else { - return false; - } - } - } - - @Override - public Map next() { - if (hasNext()) { - return currentPageResultIterator.next(); - } else { - throw new NoSuchElementException(); - } - } - } - - private static final Gson GSON = new Gson(); - - private String marketoEndpoint; - private MarketoToken token; - - public Marketo(String marketoEndpoint, String clientId, String clientSecret) { - this.marketoEndpoint = marketoEndpoint; - token = getToken(marketoEndpoint, clientId, clientSecret); - } - - public Marketo(String marketoEndpoint, MarketoToken token) { - this.marketoEndpoint = marketoEndpoint; - this.token = token; - } - - public MarketoPage getPage(String queryUrl) { - return get(queryUrl, MarketoPage.class); - } - - public MarketoPage getNextPage(MarketoPage page, String queryUrl) { - if (!Strings.isNullOrEmpty(page.getNextPageToken())) { - return getPage(queryUrl + "&nextPageToken=" + page.getNextPageToken()); - } - return null; - } - - public MarketoPageIterator iteratePage(String queryUrl) { - return new MarketoPageIterator(getPage(queryUrl), this, queryUrl); - } - - private String appendToken(String requestUrl) { - if (!requestUrl.contains("access_token")) { - if (requestUrl.contains("?")) { - return requestUrl + "&access_token=" + token.getAccessToken(); - } else { - return requestUrl + "?access_token=" + token.getAccessToken(); - } - } - return requestUrl; - } - - public T get(String queryUrl, Class cls) { - return doGet(appendToken(marketoEndpoint + queryUrl), cls); - } - - public static MarketoToken getToken(String marketoEndpoint, String clientId, String clientSecret) { - String marketoIdURL = marketoEndpoint + "/identity"; - String idEndpoint = marketoIdURL + "/oauth/token?grant_type=client_credentials&client_id=" - + clientId + "&client_secret=" + clientSecret; - - return doGet(idEndpoint, MarketoToken.class); - } - - - private static T doGet(String queryUrl, Class cls) { - try { - URL url = new URL(queryUrl); - HttpsURLConnection urlConn = (HttpsURLConnection) url.openConnection(); - urlConn.setRequestMethod("GET"); - urlConn.setRequestProperty("accept", "application/json"); - int responseCode = urlConn.getResponseCode(); - if (responseCode == 200) { - InputStream inStream = urlConn.getInputStream(); - Reader reader = new InputStreamReader(inStream); - return GSON.fromJson(reader, cls); - } else { - throw new IOException("Status: " + responseCode); - } - } catch (IOException e) { - throw new RuntimeException(e); - } - } -} diff --git a/src/main/java/io/cdap/plugin/marketo/common/MarketoEntity.java b/src/main/java/io/cdap/plugin/marketo/common/MarketoEntity.java deleted file mode 100644 index 111f239..0000000 --- a/src/main/java/io/cdap/plugin/marketo/common/MarketoEntity.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright © 2019 Cask Data, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package io.cdap.plugin.marketo.common; - -import java.util.Arrays; - -/** - * Marketo Entity types. - */ -public enum MarketoEntity { - Leads("Leads", "GET /rest/v1/leads.json", "/rest/v1/leads/describe.json"), - Campaigns("Campaigns", "/rest/v1/campaigns.json", null), - Companies("Companies", "/rest/v1/companies.json", "/rest/v1/companies/describe.json"); - - private String name; - private String getEndpoint; - private String describeEndpoint; - - MarketoEntity(String name, String getEndpoint, String describeEndpoint) { - this.name = name; - this.getEndpoint = getEndpoint; - this.describeEndpoint = describeEndpoint; - } - - public String getName() { - return name; - } - - public String getGetEndpoint() { - return getEndpoint; - } - - public String getDescribeEndpoint() { - return describeEndpoint; - } - - public static MarketoEntity fromString(String marketoEntity) { - return Arrays.stream(MarketoEntity.values()) - .filter(entity -> entity.getName().equals(marketoEntity)).findFirst() - .orElseThrow(() -> new RuntimeException(String.format("'%s' is not a valid Marketo entity", marketoEntity))); - } -} diff --git a/src/main/java/io/cdap/plugin/marketo/common/MarketoSchemaReader.java b/src/main/java/io/cdap/plugin/marketo/common/MarketoSchemaReader.java deleted file mode 100644 index 6d4996b..0000000 --- a/src/main/java/io/cdap/plugin/marketo/common/MarketoSchemaReader.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright © 2019 Cask Data, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package io.cdap.plugin.marketo.common; - -import io.cdap.cdap.api.data.schema.Schema; -import io.cdap.plugin.marketo.common.response.describe.DescribeResponse; -import io.cdap.plugin.marketo.common.response.describe.DescribeResponseLeads; -import io.cdap.plugin.marketo.common.response.describe.Field; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * Generates schema for given entity. - */ -public class MarketoSchemaReader { - // TODO fill predefined schemas - private static final Map PREDEFINED_SCHEMAS = new HashMap<>(); - - public static Schema getSchemaForEntity(MarketoEntity entity, Marketo marketo) { - if (entity.getDescribeEndpoint() != null) { - List entityFields; - - if (entity == MarketoEntity.Leads) { - DescribeResponseLeads leadsDescribe = marketo.get(entity.getDescribeEndpoint(), - DescribeResponseLeads.class); - entityFields = leadsDescribe.getFields(); - } else { - DescribeResponse describe = marketo.get(entity.getDescribeEndpoint(), DescribeResponse.class); - entityFields = describe.getFields(); - } - - List schemaFields = entityFields.stream() - //TODO map types here - .map(field -> Schema.Field.of(field.getName(), Schema.nullableOf(Schema.of(Schema.Type.STRING)))) - .collect(Collectors.toList()); - return Schema.recordOf(entity.getName(), schemaFields); - } else if (PREDEFINED_SCHEMAS.containsKey(entity)) { - return PREDEFINED_SCHEMAS.get(entity); - } - throw new RuntimeException(String.format("Unable to get schema for entity '%s'", entity.getName())); - } - -} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/HttpHelper.java b/src/main/java/io/cdap/plugin/marketo/common/api/HttpHelper.java new file mode 100644 index 0000000..ba46efd --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/HttpHelper.java @@ -0,0 +1,106 @@ +package io.cdap.plugin.marketo.common.api; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import io.cdap.plugin.marketo.common.api.entities.BaseResponse; +import org.apache.commons.io.Charsets; +import org.apache.commons.io.IOUtils; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.Reader; +import java.net.URL; +import java.util.function.Function; +import javax.net.ssl.HttpsURLConnection; + +/** + * Helper class with http routines. + */ +class HttpHelper { + private static final Gson GSON = new Gson(); + + static String doGet(String queryUrl) { + return HttpHelper.doGet(queryUrl, inputStream -> { + try { + return IOUtils.toString(inputStream, Charsets.UTF_8); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + static T doGet(String queryUrl, Class cls) { + return doGet(queryUrl, inputStream -> { + Reader reader = new InputStreamReader(inputStream); + return GSON.fromJson(reader, cls); + }); + } + + static T doGet(String queryUrl, TypeToken pageTypeToken) { + return doGet(queryUrl, inputStream -> { + Reader reader = new InputStreamReader(inputStream); + return GSON.fromJson(reader, pageTypeToken.getType()); + }); + } + + private static T doGet(String queryUrl, Function transformer) { + try { + URL url = new URL(queryUrl); + HttpsURLConnection httpConnection = (HttpsURLConnection) url.openConnection(); + httpConnection.setRequestMethod("GET"); + httpConnection.setRequestProperty("accept", "application/json"); + int responseCode = httpConnection.getResponseCode(); + if (responseCode == 200) { + InputStream inStream = httpConnection.getInputStream(); + return transformer.apply(inStream); + } else { + throw new IOException(String.format("Failed to make http request, code: %s, message: %s", + responseCode, getConnectionError(httpConnection))); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static String getConnectionError(HttpsURLConnection connection) throws IOException { + InputStream inStream = connection.getErrorStream(); + return IOUtils.toString(inStream, Charsets.UTF_8); + } + + static T doPost(String queryUrl, R body, Class requestCls, + Class responseCls) { + try { + URL url = new URL(queryUrl); + HttpsURLConnection urlConn = (HttpsURLConnection) url.openConnection(); + urlConn.setRequestMethod("POST"); + urlConn.setRequestProperty("accept", "application/json"); + + if (body != null && requestCls != null) { + urlConn.setDoOutput(true); + byte[] requestData = GSON.toJson(body, requestCls).getBytes(); + urlConn.setFixedLengthStreamingMode(requestData.length); + urlConn.setRequestProperty("Content-Type", "application/json"); + urlConn.connect(); + try (OutputStream os = urlConn.getOutputStream()) { + os.write(requestData); + } + } else { + urlConn.connect(); + } + + int responseCode = urlConn.getResponseCode(); + + if (responseCode == 200) { + InputStream inStream = urlConn.getInputStream(); + Reader reader = new InputStreamReader(inStream); + return GSON.fromJson(reader, responseCls); + } else { + throw new IOException("Status: " + responseCode); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java b/src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java new file mode 100644 index 0000000..e869f76 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java @@ -0,0 +1,63 @@ +package io.cdap.plugin.marketo.common.api; + +import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExport; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.List; + +/** + * Leads export job. + */ +public class LeadsExportJob { + private static final Logger LOG = LoggerFactory.getLogger(LeadsExportJob.class); + + private static final List WAITABLE_STATE = Arrays.asList("Queued", "Processing"); + private static final List COMPLETED_STATUS = Arrays.asList("Canceled", "Completed", "Failed"); + private String jobId; + private LeadsExport.ExportResponse last; + private Marketo marketo; + + public LeadsExportJob(LeadsExport lastStatus, Marketo marketo) { + this.jobId = lastStatus.singleExport().getExportId(); + this.last = lastStatus.singleExport(); + this.marketo = marketo; + LOG.info("Created bulk lead export job with id '{}'", this.jobId); + } + + public String getStatus() { + return last.getStatus(); + } + + public void waitCompletion() throws InterruptedException { + if (!WAITABLE_STATE.contains(getStatus())) { + throw new IllegalStateException("Job must be enqueued before waiting for completion."); + } + + while (!COMPLETED_STATUS.contains(getStatus())) { + LeadsExport currentResp = marketo.get(String.format(Urls.BULK_EXPORT_LEADS_STATUS, jobId), LeadsExport.class); + LeadsExport.ExportResponse current = currentResp.singleExport(); + String previousStatus = getStatus(); + String currentStatus = current.getStatus(); + if (!currentStatus.equals(previousStatus)) { + LOG.info("Bulk lead export job with id '{}' changed status from '{}' to '{}'", jobId, previousStatus, + currentStatus); + } + last = current; + Thread.sleep(30 * 1000); + } + LOG.info("Bulk lead export job with id '{}' finished with status '{}'", jobId, getStatus()); + } + + public void enqueue() { + last = marketo.post(String.format(Urls.BULK_EXPORT_LEADS_ENQUEUE, jobId), + null, null, LeadsExport.class).singleExport(); + + LOG.info("Bulk lead export job with id '{}' enqueued", jobId); + } + + public String getFile() { + return marketo.get(String.format(Urls.BULK_EXPORT_LEADS_FILE, jobId)); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java b/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java new file mode 100644 index 0000000..2ef7a16 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java @@ -0,0 +1,121 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api; + +import com.google.common.base.Strings; +import com.google.gson.reflect.TypeToken; +import io.cdap.plugin.marketo.common.api.entities.BaseResponse; +import io.cdap.plugin.marketo.common.api.entities.MarketoToken; +import io.cdap.plugin.marketo.common.api.entities.leads.LeadsDescribe; +import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExport; +import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExportRequest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +/** + * Helper class to perform Marketo Api operations. + */ +public class Marketo { + + private static final Logger LOG = LoggerFactory.getLogger(Marketo.class); + private static final TypeToken LEADS_DESCRIBE_TYPE_TOKEN = new TypeToken() { + }; + private String marketoEndpoint; + private MarketoToken token; + + public Marketo(String marketoEndpoint, String clientId, String clientSecret) { + this.marketoEndpoint = marketoEndpoint; + token = getToken(marketoEndpoint, clientId, clientSecret); + } + + public Marketo(String marketoEndpoint, MarketoToken token) { + this.marketoEndpoint = marketoEndpoint; + this.token = token; + } + + public List describeLeads() { + return StreamSupport.stream(Spliterators.spliteratorUnknownSize( + iteratePage(Urls.LEADS_DESCRIBE, LEADS_DESCRIBE_TYPE_TOKEN, LeadsDescribe::getResult), + Spliterator.ORDERED), false).collect(Collectors.toList()); + } + + public LeadsExportJob exportLeads(LeadsExportRequest request) { + LeadsExport export = post(Urls.BULK_EXPORT_LEADS_CREATE, request, LeadsExportRequest.class, LeadsExport.class); + return new LeadsExportJob(export, this); + } + + T getPage(String queryUrl, TypeToken pageTypeToken) { + return get(queryUrl, pageTypeToken); + } + + T getNextPage(T currentPage, String queryUrl, TypeToken pageTypeToken) { + if (!Strings.isNullOrEmpty(currentPage.getNextPageToken())) { + return getPage(queryUrl + "&nextPageToken=" + currentPage.getNextPageToken(), pageTypeToken); + } + return null; + } + + MarketoPageIterator iteratePage(String queryUrl, + TypeToken pageTypeToken, + Function> resultsGetter) { + return new MarketoPageIterator<>(getPage(queryUrl, pageTypeToken), this, queryUrl, pageTypeToken, resultsGetter); + } + + T get(String queryUrl, TypeToken pageTypeToken) { + return HttpHelper.doGet(appendToken(marketoEndpoint + queryUrl), pageTypeToken); + } + + T get(String queryUrl, Class cls) { + return HttpHelper.doGet(appendToken(marketoEndpoint + queryUrl), cls); + } + + T post(String queryUrl, R body, Class requestCls, Class responseCls) { + return HttpHelper.doPost(appendToken(marketoEndpoint + queryUrl), body, requestCls, responseCls); + } + + String get(String queryUrl) { + return HttpHelper.doGet(queryUrl); + } + + public static MarketoToken getToken(String marketoEndpoint, String clientId, String clientSecret) { + LOG.info("Requesting marketo token"); + String marketoIdURL = marketoEndpoint + "/identity"; + String idEndpoint = marketoIdURL + "/oauth/token?grant_type=client_credentials&client_id=" + + clientId + "&client_secret=" + clientSecret; + + return HttpHelper.doGet(idEndpoint, MarketoToken.class); + } + + private String appendToken(String requestUrl) { + if (!requestUrl.contains("access_token")) { + if (requestUrl.contains("?")) { + return requestUrl + "&access_token=" + token.getAccessToken(); + } else { + return requestUrl + "?access_token=" + token.getAccessToken(); + } + } + return requestUrl; + } + +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/MarketoPageIterator.java b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoPageIterator.java new file mode 100644 index 0000000..fc1a0a9 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoPageIterator.java @@ -0,0 +1,59 @@ +package io.cdap.plugin.marketo.common.api; + +import com.google.gson.reflect.TypeToken; +import io.cdap.plugin.marketo.common.api.entities.BaseResponse; + +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.function.Function; + +/** + * Marketo page iterator. + * + * @param type of page response + * @param type of page item entity + */ +public class MarketoPageIterator implements Iterator { + private T currentPage; + private Marketo marketo; + private String queryUrl; + private TypeToken pageTypeToken; + private Function> resultsGetter; + private Iterator currentPageResultIterator; + + MarketoPageIterator(T page, Marketo marketo, String queryUrl, TypeToken pageTypeToken, + Function> resultsGetter) { + this.currentPage = page; + this.marketo = marketo; + this.queryUrl = queryUrl; + this.pageTypeToken = pageTypeToken; + this.resultsGetter = resultsGetter; + currentPageResultIterator = resultsGetter.apply(this.currentPage).iterator(); + } + + @Override + public boolean hasNext() { + if (currentPageResultIterator.hasNext()) { + return true; + } else { + T nextPage = marketo.getNextPage(currentPage, queryUrl, pageTypeToken); + if (nextPage != null) { + currentPage = nextPage; + currentPageResultIterator = resultsGetter.apply(this.currentPage).iterator(); + return hasNext(); + } else { + return false; + } + } + } + + @Override + public I next() { + if (hasNext()) { + return currentPageResultIterator.next(); + } else { + throw new NoSuchElementException(); + } + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Urls.java b/src/main/java/io/cdap/plugin/marketo/common/api/Urls.java new file mode 100644 index 0000000..dc0ae5f --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Urls.java @@ -0,0 +1,12 @@ +package io.cdap.plugin.marketo.common.api; + +/** + * Marketo API urls. + */ +public class Urls { + public static final String BULK_EXPORT_LEADS_CREATE = "/bulk/v1/leads/export/create.json"; + public static final String BULK_EXPORT_LEADS_ENQUEUE = "/bulk/v1/leads/export/%s/enqueue.json"; + public static final String BULK_EXPORT_LEADS_STATUS = "/bulk/v1/leads/export/%s/status.json"; + public static final String BULK_EXPORT_LEADS_FILE = "/bulk/v1/leads/export/%s/file.json"; + public static final String LEADS_DESCRIBE = "/rest/v1/leads/describe.json"; +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/BaseResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/BaseResponse.java new file mode 100644 index 0000000..368fe0e --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/BaseResponse.java @@ -0,0 +1,41 @@ +package io.cdap.plugin.marketo.common.api.entities; + +import java.util.Collections; +import java.util.List; + +/** + * Represents common parts for all responses. + */ +public class BaseResponse { + + private boolean success = false; + private List errors = Collections.emptyList(); + private List warnings = Collections.emptyList(); + private String requestId; + private boolean moreResult = false; + private String nextPageToken; + + public boolean isSuccess() { + return success; + } + + public List getErrors() { + return errors; + } + + public List getWarnings() { + return warnings; + } + + public String getRequestId() { + return requestId; + } + + public boolean isMoreResult() { + return moreResult; + } + + public String getNextPageToken() { + return nextPageToken; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/Error.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/Error.java new file mode 100644 index 0000000..681577d --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/Error.java @@ -0,0 +1,22 @@ +package io.cdap.plugin.marketo.common.api.entities; + +/** + * Represents error message. + */ +public class Error { + private int code; + private String message; + + public int getCode() { + return code; + } + + public String getMessage() { + return message; + } + + @Override + public String toString() { + return String.format("code: %d, message: %s", code, message); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/MarketoToken.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/MarketoToken.java similarity index 93% rename from src/main/java/io/cdap/plugin/marketo/common/MarketoToken.java rename to src/main/java/io/cdap/plugin/marketo/common/api/entities/MarketoToken.java index 6f7c235..258d93f 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/MarketoToken.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/MarketoToken.java @@ -14,12 +14,12 @@ * the License. */ -package io.cdap.plugin.marketo.common; +package io.cdap.plugin.marketo.common.api.entities; import com.google.gson.annotations.SerializedName; /** - * Marketo Token holder. + * Represents marketo token response. */ public class MarketoToken { @SerializedName("access_token") diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/Warning.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/Warning.java new file mode 100644 index 0000000..562ffa1 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/Warning.java @@ -0,0 +1,22 @@ +package io.cdap.plugin.marketo.common.api.entities; + +/** + * Represents warning message. + */ +public class Warning { + private int code; + private String message; + + public int getCode() { + return code; + } + + public String getMessage() { + return message; + } + + @Override + public String toString() { + return String.format("code: %d, message: %s", code, message); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsDescribe.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsDescribe.java new file mode 100644 index 0000000..e1c832b --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsDescribe.java @@ -0,0 +1,69 @@ +package io.cdap.plugin.marketo.common.api.entities.leads; + +import io.cdap.plugin.marketo.common.api.entities.BaseResponse; + +import java.util.Collections; +import java.util.List; + +/** + * Represents leads describe response. + */ +public class LeadsDescribe extends BaseResponse { + /** + * Represents lead field description. + */ + public static class LeadAttribute { + String dataType; + String displayName; + int id; + int length; + LeadMapAttribute rest; + LeadMapAttribute soap; + + public String getDataType() { + return dataType; + } + + public String getDisplayName() { + return displayName; + } + + public int getId() { + return id; + } + + public int getLength() { + return length; + } + + public LeadMapAttribute getRest() { + return rest; + } + + public LeadMapAttribute getSoap() { + return soap; + } + } + + /** + * Represents leads field name. + */ + public static class LeadMapAttribute { + String name; + boolean readOnly = true; + + public String getName() { + return name; + } + + public boolean isReadOnly() { + return readOnly; + } + } + + List result = Collections.emptyList(); + + public List getResult() { + return result; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExport.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExport.java new file mode 100644 index 0000000..877630b --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExport.java @@ -0,0 +1,82 @@ +package io.cdap.plugin.marketo.common.api.entities.leads; + +import io.cdap.plugin.marketo.common.api.entities.BaseResponse; + +import java.util.Collections; +import java.util.List; + +/** + * Represents leads bulk export response. + */ +public class LeadsExport extends BaseResponse { + /** + * Represents export response item. + */ + public static class ExportResponse { + String createdAt; + String errorMsg; + String exportId; + int fileSize; + String fileChecksum; + String finishedAt; + String format; + int numberOfRecords; + String queuedAt; + String startedAt; + String status; + + public String getCreatedAt() { + return createdAt; + } + + public String getErrorMsg() { + return errorMsg; + } + + public String getExportId() { + return exportId; + } + + public int getFileSize() { + return fileSize; + } + + public String getFileChecksum() { + return fileChecksum; + } + + public String getFinishedAt() { + return finishedAt; + } + + public String getFormat() { + return format; + } + + public int getNumberOfRecords() { + return numberOfRecords; + } + + public String getQueuedAt() { + return queuedAt; + } + + public String getStartedAt() { + return startedAt; + } + + public String getStatus() { + return status; + } + } + + List result = Collections.emptyList(); + + public ExportResponse singleExport() { + if (result.size() != 1) { + throw new IllegalStateException("Expected single export job result."); + } + return result.get(0); + } + +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportRequest.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportRequest.java new file mode 100644 index 0000000..d46f0d4 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportRequest.java @@ -0,0 +1,107 @@ +package io.cdap.plugin.marketo.common.api.entities.leads; + +import java.util.List; +import java.util.Map; + +/** + * Represents leads bulk export request. + */ +public class LeadsExportRequest { + /** + * Represents date range. + */ + public static class DateRange { + String endAt = null; + String startAt = null; + + public DateRange(String startAt, String endAt) { + this.endAt = endAt; + this.startAt = startAt; + } + } + + /** + * Represents request filter. + */ + public static class ExportLeadFilter { + DateRange createdAt = null; + Integer smartListId = null; + String smartListName = null; + Integer staticListId = null; + Integer staticListName = null; + DateRange updatedAt = null; + + /** + * Builder for ExportLeadFilter. + */ + public static class Builder { + private DateRange createdAt = null; + private Integer smartListId = null; + private String smartListName = null; + private Integer staticListId = null; + private Integer staticListName = null; + private DateRange updatedAt = null; + + public Builder() { + } + + public Builder createdAt(DateRange createdAt) { + this.createdAt = createdAt; + return Builder.this; + } + + public Builder smartListId(Integer smartListId) { + this.smartListId = smartListId; + return Builder.this; + } + + public Builder smartListName(String smartListName) { + this.smartListName = smartListName; + return Builder.this; + } + + public Builder staticListId(Integer staticListId) { + this.staticListId = staticListId; + return Builder.this; + } + + public Builder staticListName(Integer staticListName) { + this.staticListName = staticListName; + return Builder.this; + } + + public Builder updatedAt(DateRange updatedAt) { + this.updatedAt = updatedAt; + return Builder.this; + } + + public ExportLeadFilter build() { + + return new ExportLeadFilter(this); + } + } + + private ExportLeadFilter(Builder builder) { + this.createdAt = builder.createdAt; + this.smartListId = builder.smartListId; + this.smartListName = builder.smartListName; + this.staticListId = builder.staticListId; + this.staticListName = builder.staticListName; + this.updatedAt = builder.updatedAt; + } + + public static Builder builder() { + return new Builder(); + } + } + + Map columnHeaderNames = null; + List fields = null; + ExportLeadFilter filter = null; + String format = "CSV"; + + public LeadsExportRequest(List fields, ExportLeadFilter filter) { + this.fields = fields; + this.filter = filter; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/response/MarketoPage.java b/src/main/java/io/cdap/plugin/marketo/common/response/MarketoPage.java deleted file mode 100644 index 5311e7f..0000000 --- a/src/main/java/io/cdap/plugin/marketo/common/response/MarketoPage.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright © 2019 Cask Data, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package io.cdap.plugin.marketo.common.response; - -import java.util.Collections; -import java.util.List; -import java.util.Map; - -/** - * Marketo page response. - */ -public class MarketoPage { - /** - * Error or warn message description. - */ - public static class Description { - private String code; - private String message; - - public String getCode() { - return code; - } - - public String getMessage() { - return message; - } - } - - private Boolean moreResult = null; - private String nextPageToken = null; - private String requestId = null; - private Boolean success = null; - - private List errors = Collections.emptyList(); - private List warnings = Collections.emptyList(); - - private List> result = Collections.emptyList(); - - public Boolean hasMoreResults() { - return moreResult; - } - - public String getNextPageToken() { - return nextPageToken; - } - - public String getRequestId() { - return requestId; - } - - public Boolean isSuccess() { - return success; - } - - public List getErrors() { - return errors; - } - - public List getWarnings() { - return warnings; - } - - public List> getResults() { - return result; - } -} diff --git a/src/main/java/io/cdap/plugin/marketo/common/response/describe/DescribeResponse.java b/src/main/java/io/cdap/plugin/marketo/common/response/describe/DescribeResponse.java deleted file mode 100644 index 23cc88f..0000000 --- a/src/main/java/io/cdap/plugin/marketo/common/response/describe/DescribeResponse.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright © 2019 Cask Data, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package io.cdap.plugin.marketo.common.response.describe; - -import java.util.Collections; -import java.util.List; - -/** - * Regular describe response. - * Fields located in first object in 'result' field. - */ -public class DescribeResponse { - /** - * Result holder. - */ - public static class Result { - List fields = Collections.emptyList(); - } - - private String requestId = null; - private Boolean success = null; - - private List result = Collections.emptyList(); - - public List getFields() { - if (result.size() != 1) { - throw new RuntimeException("Expected to have one result."); - } - return result.get(0).fields; - } -} diff --git a/src/main/java/io/cdap/plugin/marketo/common/response/describe/DescribeResponseLeads.java b/src/main/java/io/cdap/plugin/marketo/common/response/describe/DescribeResponseLeads.java deleted file mode 100644 index e5eefea..0000000 --- a/src/main/java/io/cdap/plugin/marketo/common/response/describe/DescribeResponseLeads.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright © 2019 Cask Data, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package io.cdap.plugin.marketo.common.response.describe; - -import java.util.Collections; -import java.util.List; - -/** - * Describe response for Leads-Leads entity. - * Fields here is directly located in 'result' field of response. - */ -public class DescribeResponseLeads { - private String requestId = null; - private Boolean success = null; - - List result = Collections.emptyList(); - - public List getFields() { - return result; - } -} diff --git a/src/main/java/io/cdap/plugin/marketo/common/response/describe/Field.java b/src/main/java/io/cdap/plugin/marketo/common/response/describe/Field.java deleted file mode 100644 index 7086298..0000000 --- a/src/main/java/io/cdap/plugin/marketo/common/response/describe/Field.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright © 2019 Cask Data, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package io.cdap.plugin.marketo.common.response.describe; - -/** - * Marketo field description. - */ -public class Field { - /** - * Soap or rest descriptor. - */ - public static class FieldDescription { - String name; - Boolean readOnly; - } - - String id = null; - private String name = null; - String displayName = null; - String dataType = null; - int length = -1; - Boolean updateable = null; - FieldDescription rest = null; - FieldDescription soap = null; - - public String getName() { - if (name == null) { - if (rest == null) { - throw new RuntimeException("Failed to get name for field."); - } else { - return rest.name; - } - } else { - return name; - } - } -} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormatProvider.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormatProvider.java index b1bebb7..8a70ed1 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormatProvider.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormatProvider.java @@ -28,12 +28,12 @@ * InputFormatProvider used by cdap to provide configurations to mapreduce job */ public class MarketoInputFormatProvider implements InputFormatProvider { - public static final String PROPERTY_CONFIG_JSON = "cdap.marketo.entity.config"; + public static final String PROPERTY_CONFIG_JSON = "cdap.marketo.reporter.config"; private static final Gson gson = new GsonBuilder().create(); private final Map conf; - MarketoInputFormatProvider(MarketoBatchSourceConfig config) { + MarketoInputFormatProvider(MarketoReportingSourceConfig config) { this.conf = Collections.unmodifiableMap(new HashMap() {{ put(PROPERTY_CONFIG_JSON, gson.toJson(config)); }}); diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java index 9d36451..a3b0e70 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java @@ -18,37 +18,71 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import io.cdap.plugin.marketo.common.Marketo; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.marketo.common.api.LeadsExportJob; +import io.cdap.plugin.marketo.common.api.Marketo; +import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExportRequest; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.RecordReader; import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; +import java.io.StringReader; +import java.util.Iterator; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * RecordReader implementation, which reads events from Marketo api. */ -public class MarketoRecordReader extends RecordReader> { +public class MarketoRecordReader extends RecordReader> { + private static final Logger LOG = LoggerFactory.getLogger(MarketoRecordReader.class); private static final Gson GSON = new GsonBuilder().create(); - private Marketo.MarketoPageIterator iterator; - private Map current = null; + private Map current = null; + private Iterator iterator = null; @Override public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException { Configuration conf = taskAttemptContext.getConfiguration(); String configJson = conf.get(MarketoInputFormatProvider.PROPERTY_CONFIG_JSON); - MarketoBatchSourceConfig config = GSON.fromJson(configJson, MarketoBatchSourceConfig.class); + MarketoReportingSourceConfig config = GSON.fromJson(configJson, MarketoReportingSourceConfig.class); - iterator = config.getMarketo().iteratePage(config.getEntityType().getGetEndpoint()); + Marketo marketo = config.getMarketo(); + + List fields = config.getSchema().getFields().stream() + .map(Schema.Field::getName).collect(Collectors.toList()); + LeadsExportRequest.ExportLeadFilter filter = LeadsExportRequest.ExportLeadFilter.builder() + .createdAt(new LeadsExportRequest.DateRange(config.getStartDate(), config.getEndDate())).build(); + LeadsExportRequest request = new LeadsExportRequest(fields, filter); + + LeadsExportJob job = marketo.exportLeads(request); + job.enqueue(); + try { + job.waitCompletion(); + } catch (InterruptedException ex) { + throw new RuntimeException(ex); + } + + String data = job.getFile(); + LOG.info(data); + CSVParser parser = CSVFormat.DEFAULT.withHeader().parse(new StringReader(data)); + iterator = parser.iterator(); +// iterator = config.getMarketo().iteratePage(config.getEntityType().getGetEndpoint()); } @Override public boolean nextKeyValue() { if (iterator.hasNext()) { - current = iterator.next(); + current = iterator.next().toMap(); + LOG.debug("Got record '{}'", current.toString()); return true; } return false; @@ -60,7 +94,7 @@ public NullWritable getCurrentKey() { } @Override - public Map getCurrentValue() { + public Map getCurrentValue() { return current; } diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoBatchSource.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingPlugin.java similarity index 83% rename from src/main/java/io/cdap/plugin/marketo/source/batch/MarketoBatchSource.java rename to src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingPlugin.java index ad6545e..e7949e7 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoBatchSource.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingPlugin.java @@ -39,14 +39,14 @@ * Plugin that reads entities from Marketo api. */ @Plugin(type = BatchSource.PLUGIN_TYPE) -@Name(MarketoBatchSource.NAME) +@Name(MarketoReportingPlugin.NAME) @Description("Reads entities from Marketo.") -public class MarketoBatchSource extends BatchSource, StructuredRecord> { - public static final String NAME = "MarketoEntityPlugin"; +public class MarketoReportingPlugin extends BatchSource, StructuredRecord> { + public static final String NAME = "MarketoReportingPlugin"; - private final MarketoBatchSourceConfig config; + private final MarketoReportingSourceConfig config; - public MarketoBatchSource(MarketoBatchSourceConfig config) { + public MarketoReportingPlugin(MarketoReportingSourceConfig config) { this.config = config; } @@ -71,17 +71,17 @@ public void prepareRun(BatchSourceContext batchSourceContext) { } @Override - public void transform(KeyValue> input, Emitter emitter) { + public void transform(KeyValue> input, Emitter emitter) { StructuredRecord.Builder builder = StructuredRecord.builder(config.getSchema()); - Map inputMap = input.getValue(); + Map inputMap = input.getValue(); config.getSchema().getFields().forEach( field -> { if (inputMap.containsKey(field.getName())) { - //TODO map fields, also refactor io.cdap.plugin.marketo.common.MarketoSchemaReader.getSchemaForEntity builder.set(field.getName(), String.valueOf(inputMap.remove(field.getName()))); } } ); + emitter.emit(builder.build()); } private void validateConfiguration(FailureCollector failureCollector) { diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoBatchSourceConfig.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java similarity index 82% rename from src/main/java/io/cdap/plugin/marketo/source/batch/MarketoBatchSourceConfig.java rename to src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java index f7f16f2..f139ddf 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoBatchSourceConfig.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java @@ -21,15 +21,17 @@ import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.plugin.common.ReferencePluginConfig; -import io.cdap.plugin.marketo.common.Marketo; -import io.cdap.plugin.marketo.common.MarketoEntity; -import io.cdap.plugin.marketo.common.MarketoSchemaReader; -import io.cdap.plugin.marketo.common.MarketoToken; +import io.cdap.plugin.marketo.common.api.Marketo; +import io.cdap.plugin.marketo.common.api.entities.MarketoToken; + +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; /** * Provides all required configuration for reading Marketo entities. */ -public class MarketoBatchSourceConfig extends ReferencePluginConfig { +public class MarketoReportingSourceConfig extends ReferencePluginConfig { public static final String PROPERTY_ENTITY_NAME = "entityName"; public static final String PROPERTY_CLIENT_ID = "clientId"; public static final String PROPERTY_CLIENT_SECRET = "clientSecret"; @@ -90,13 +92,23 @@ public class MarketoBatchSourceConfig extends ReferencePluginConfig { private transient Schema schema = null; private transient Marketo marketo = null; - public MarketoBatchSourceConfig(String referenceName) { + public MarketoReportingSourceConfig(String referenceName) { super(referenceName); } public Schema getSchema() { if (schema == null) { - schema = MarketoSchemaReader.getSchemaForEntity(getEntityType(), getMarketo()); + List fields = getMarketo().describeLeads().stream().map( + leadAttribute -> { + if (leadAttribute.getRest() != null) { + return Schema.Field.of(leadAttribute.getRest().getName(), Schema.nullableOf(Schema.of(Schema.Type.STRING))); + } else { + return null; + } + } + ).filter(Objects::nonNull).collect(Collectors.toList()); + + schema = Schema.recordOf("LeadsRecord", fields); } return schema; } @@ -108,9 +120,6 @@ public Marketo getMarketo() { return marketo; } - public MarketoEntity getEntityType() { - return MarketoEntity.fromString(entityName); - } public String getClientId() { return clientId; diff --git a/widgets/MarketoReportingPlugin-batchsource.json b/widgets/MarketoReportingPlugin-batchsource.json new file mode 100644 index 0000000..8e4ee54 --- /dev/null +++ b/widgets/MarketoReportingPlugin-batchsource.json @@ -0,0 +1,28 @@ +{ + "metadata": { + "spec-version": "1.0" + }, + "display-name": "Marketo Reporting", + "configuration-groups": [ + { + "label": "General", + "properties": [ + { + "widget-type": "textbox", + "label": "Reference Name", + "name": "referenceName" + } + ] + }, + { + "label": "Advanced", + "properties": [ ] + } + ], + "outputs": [ + { + "widget-type": "non-editable-schema-editor", + "schema": { } + } + ] +} \ No newline at end of file From fd5b4048e1aa49832874d4acae540b383cea704b Mon Sep 17 00:00:00 2001 From: Yevhenii Chekanskyi Date: Mon, 25 Nov 2019 21:13:38 +0200 Subject: [PATCH 03/12] PLUGIN-75 Marketo Plugin - refactoring. --- pom.xml | 18 ++ .../plugin/marketo/common/api/Helpers.java | 51 +++++ .../plugin/marketo/common/api/HttpHelper.java | 106 ---------- .../marketo/common/api/LeadsExportJob.java | 14 +- .../plugin/marketo/common/api/Marketo.java | 192 ++++++++++++++---- .../common/api/MarketoPageIterator.java | 9 +- .../common/api/entities/BaseResponse.java | 28 ++- .../marketo/common/api/entities/Error.java | 8 + .../marketo/common/api/entities/Warning.java | 8 + .../batch/MarketoReportingSourceConfig.java | 9 +- .../marketo/common/api/MarketoTest.java | 187 +++++++++++++++++ 11 files changed, 463 insertions(+), 167 deletions(-) create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java delete mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/HttpHelper.java create mode 100644 src/test/java/io/cdap/plugin/marketo/common/api/MarketoTest.java diff --git a/pom.xml b/pom.xml index fb9a529..d237f88 100644 --- a/pom.xml +++ b/pom.xml @@ -22,6 +22,7 @@ 6.1.0-SNAPSHOT 2.3.0 + 4.5.9 2.3.0-SNAPSHOT 2.1.3 @@ -238,6 +239,11 @@ + + org.apache.httpcomponents + httpclient + ${httpcomponents.version} + com.google.code.gson gson @@ -254,12 +260,24 @@ 2.6 + + org.slf4j + slf4j-simple + 1.7.29 + + junit junit 4.12 test + + com.github.tomakehurst + wiremock-jre8-standalone + 2.25.1 + test + diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java b/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java new file mode 100644 index 0000000..78c77ad --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java @@ -0,0 +1,51 @@ +package io.cdap.plugin.marketo.common.api; + +import com.google.common.base.Strings; +import org.apache.commons.io.IOUtils; +import org.apache.http.NameValuePair; +import org.apache.http.client.utils.URIBuilder; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.util.List; + +public class Helpers { + public static String streamToString(InputStream inputStream) { + try { + return IOUtils.toString(inputStream, StandardCharsets.UTF_8); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public static T streamToObject(InputStream inputStream, Class cls) { + return Marketo.GSON.fromJson(new InputStreamReader(inputStream), cls); + } + + public static RuntimeException failForUri(String method, URI uri, Exception ex) { + String message = ex.getMessage(); + if (Strings.isNullOrEmpty(message)) { + if (ex.getCause() != null) { + message = ex.getCause().getMessage(); + if (Strings.isNullOrEmpty(message)) { + message = "failed to make request"; + } + } + } + + URIBuilder uriBuilder = new URIBuilder(uri); + List queryParameters = uriBuilder.getQueryParams(); + queryParameters.removeIf(queryParameter -> queryParameter.getName().equals("access_token")); + uriBuilder.setParameters(queryParameters); + try { + String uriString = uriBuilder.build().toString(); + return new RuntimeException(String.format("Failed %s %s - %s", method, uriString, message)); + } catch (URISyntaxException e) { + return new RuntimeException(message); + } + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/HttpHelper.java b/src/main/java/io/cdap/plugin/marketo/common/api/HttpHelper.java deleted file mode 100644 index ba46efd..0000000 --- a/src/main/java/io/cdap/plugin/marketo/common/api/HttpHelper.java +++ /dev/null @@ -1,106 +0,0 @@ -package io.cdap.plugin.marketo.common.api; - -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; -import io.cdap.plugin.marketo.common.api.entities.BaseResponse; -import org.apache.commons.io.Charsets; -import org.apache.commons.io.IOUtils; - -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.io.Reader; -import java.net.URL; -import java.util.function.Function; -import javax.net.ssl.HttpsURLConnection; - -/** - * Helper class with http routines. - */ -class HttpHelper { - private static final Gson GSON = new Gson(); - - static String doGet(String queryUrl) { - return HttpHelper.doGet(queryUrl, inputStream -> { - try { - return IOUtils.toString(inputStream, Charsets.UTF_8); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - } - - static T doGet(String queryUrl, Class cls) { - return doGet(queryUrl, inputStream -> { - Reader reader = new InputStreamReader(inputStream); - return GSON.fromJson(reader, cls); - }); - } - - static T doGet(String queryUrl, TypeToken pageTypeToken) { - return doGet(queryUrl, inputStream -> { - Reader reader = new InputStreamReader(inputStream); - return GSON.fromJson(reader, pageTypeToken.getType()); - }); - } - - private static T doGet(String queryUrl, Function transformer) { - try { - URL url = new URL(queryUrl); - HttpsURLConnection httpConnection = (HttpsURLConnection) url.openConnection(); - httpConnection.setRequestMethod("GET"); - httpConnection.setRequestProperty("accept", "application/json"); - int responseCode = httpConnection.getResponseCode(); - if (responseCode == 200) { - InputStream inStream = httpConnection.getInputStream(); - return transformer.apply(inStream); - } else { - throw new IOException(String.format("Failed to make http request, code: %s, message: %s", - responseCode, getConnectionError(httpConnection))); - } - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private static String getConnectionError(HttpsURLConnection connection) throws IOException { - InputStream inStream = connection.getErrorStream(); - return IOUtils.toString(inStream, Charsets.UTF_8); - } - - static T doPost(String queryUrl, R body, Class requestCls, - Class responseCls) { - try { - URL url = new URL(queryUrl); - HttpsURLConnection urlConn = (HttpsURLConnection) url.openConnection(); - urlConn.setRequestMethod("POST"); - urlConn.setRequestProperty("accept", "application/json"); - - if (body != null && requestCls != null) { - urlConn.setDoOutput(true); - byte[] requestData = GSON.toJson(body, requestCls).getBytes(); - urlConn.setFixedLengthStreamingMode(requestData.length); - urlConn.setRequestProperty("Content-Type", "application/json"); - urlConn.connect(); - try (OutputStream os = urlConn.getOutputStream()) { - os.write(requestData); - } - } else { - urlConn.connect(); - } - - int responseCode = urlConn.getResponseCode(); - - if (responseCode == 200) { - InputStream inStream = urlConn.getInputStream(); - Reader reader = new InputStreamReader(inStream); - return GSON.fromJson(reader, responseCls); - } else { - throw new IOException("Status: " + responseCode); - } - } catch (IOException e) { - throw new RuntimeException(e); - } - } -} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java b/src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java index e869f76..2feaf0f 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java @@ -5,6 +5,7 @@ import org.slf4j.LoggerFactory; import java.util.Arrays; +import java.util.Collections; import java.util.List; /** @@ -12,9 +13,9 @@ */ public class LeadsExportJob { private static final Logger LOG = LoggerFactory.getLogger(LeadsExportJob.class); - private static final List WAITABLE_STATE = Arrays.asList("Queued", "Processing"); private static final List COMPLETED_STATUS = Arrays.asList("Canceled", "Completed", "Failed"); + private String jobId; private LeadsExport.ExportResponse last; private Marketo marketo; @@ -36,13 +37,15 @@ public void waitCompletion() throws InterruptedException { } while (!COMPLETED_STATUS.contains(getStatus())) { - LeadsExport currentResp = marketo.get(String.format(Urls.BULK_EXPORT_LEADS_STATUS, jobId), LeadsExport.class); + LeadsExport currentResp = marketo.validatedGet( + String.format(Urls.BULK_EXPORT_LEADS_STATUS, jobId), + Collections.emptyMap(), inputStream -> Helpers.streamToObject(inputStream, LeadsExport.class)); LeadsExport.ExportResponse current = currentResp.singleExport(); String previousStatus = getStatus(); String currentStatus = current.getStatus(); if (!currentStatus.equals(previousStatus)) { LOG.info("Bulk lead export job with id '{}' changed status from '{}' to '{}'", jobId, previousStatus, - currentStatus); + currentStatus); } last = current; Thread.sleep(30 * 1000); @@ -52,12 +55,13 @@ public void waitCompletion() throws InterruptedException { public void enqueue() { last = marketo.post(String.format(Urls.BULK_EXPORT_LEADS_ENQUEUE, jobId), - null, null, LeadsExport.class).singleExport(); + null, LeadsExport.class).singleExport(); LOG.info("Bulk lead export job with id '{}' enqueued", jobId); } public String getFile() { - return marketo.get(String.format(Urls.BULK_EXPORT_LEADS_FILE, jobId)); + return marketo.get(marketo.buildUri(String.format(Urls.BULK_EXPORT_LEADS_FILE, jobId), Collections.emptyMap()), + Helpers::streamToString); } } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java b/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java index 2ef7a16..49e569b 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java @@ -17,19 +17,41 @@ package io.cdap.plugin.marketo.common.api; import com.google.common.base.Strings; -import com.google.gson.reflect.TypeToken; +import com.google.common.collect.ImmutableMap; +import com.google.gson.Gson; import io.cdap.plugin.marketo.common.api.entities.BaseResponse; +import io.cdap.plugin.marketo.common.api.entities.Error; import io.cdap.plugin.marketo.common.api.entities.MarketoToken; import io.cdap.plugin.marketo.common.api.entities.leads.LeadsDescribe; import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExport; import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExportRequest; +import org.apache.commons.io.IOUtils; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.Spliterator; import java.util.Spliterators; import java.util.function.Function; +import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -37,85 +59,173 @@ * Helper class to perform Marketo Api operations. */ public class Marketo { - private static final Logger LOG = LoggerFactory.getLogger(Marketo.class); - private static final TypeToken LEADS_DESCRIBE_TYPE_TOKEN = new TypeToken() { - }; + static final Gson GSON = new Gson(); private String marketoEndpoint; + private String clientId; + private String clientSecret; private MarketoToken token; + private HttpClientContext httpClientContext = HttpClientContext.create(); public Marketo(String marketoEndpoint, String clientId, String clientSecret) { this.marketoEndpoint = marketoEndpoint; - token = getToken(marketoEndpoint, clientId, clientSecret); - } - - public Marketo(String marketoEndpoint, MarketoToken token) { - this.marketoEndpoint = marketoEndpoint; - this.token = token; + this.clientId = clientId; + this.clientSecret = clientSecret; + token = refreshToken(); } public List describeLeads() { return StreamSupport.stream(Spliterators.spliteratorUnknownSize( - iteratePage(Urls.LEADS_DESCRIBE, LEADS_DESCRIBE_TYPE_TOKEN, LeadsDescribe::getResult), + iteratePage(Urls.LEADS_DESCRIBE, LeadsDescribe.class, LeadsDescribe::getResult), Spliterator.ORDERED), false).collect(Collectors.toList()); } public LeadsExportJob exportLeads(LeadsExportRequest request) { - LeadsExport export = post(Urls.BULK_EXPORT_LEADS_CREATE, request, LeadsExportRequest.class, LeadsExport.class); + LeadsExport export = validatedPost(Urls.BULK_EXPORT_LEADS_CREATE, Collections.emptyMap(), + inputStream -> Helpers.streamToObject(inputStream, LeadsExport.class), + request, + GSON::toJson); return new LeadsExportJob(export, this); } - T getPage(String queryUrl, TypeToken pageTypeToken) { - return get(queryUrl, pageTypeToken); + private T getPage(String queryUrl, Class pageClass) { + return validatedGet(queryUrl, Collections.emptyMap(), + inputStream -> Helpers.streamToObject(inputStream, pageClass)); } - T getNextPage(T currentPage, String queryUrl, TypeToken pageTypeToken) { + T getNextPage(T currentPage, String queryUrl, Class pageClass) { if (!Strings.isNullOrEmpty(currentPage.getNextPageToken())) { - return getPage(queryUrl + "&nextPageToken=" + currentPage.getNextPageToken(), pageTypeToken); + return validatedGet(queryUrl, + ImmutableMap.of("nextPageToken", currentPage.getNextPageToken()), + inputStream -> Helpers.streamToObject(inputStream, pageClass)); } return null; } - MarketoPageIterator iteratePage(String queryUrl, - TypeToken pageTypeToken, - Function> resultsGetter) { - return new MarketoPageIterator<>(getPage(queryUrl, pageTypeToken), this, queryUrl, pageTypeToken, resultsGetter); + private MarketoPageIterator iteratePage(String queryUrl, + Class pageClass, + Function> resultsGetter) { + return new MarketoPageIterator<>(getPage(queryUrl, pageClass), this, queryUrl, pageClass, resultsGetter); } - T get(String queryUrl, TypeToken pageTypeToken) { - return HttpHelper.doGet(appendToken(marketoEndpoint + queryUrl), pageTypeToken); + T post(String queryUrl, R body, Class responseCls) { + return validatedPost(queryUrl, Collections.emptyMap(), + inputStream -> Helpers.streamToObject(inputStream, responseCls), + body, GSON::toJson); } - T get(String queryUrl, Class cls) { - return HttpHelper.doGet(appendToken(marketoEndpoint + queryUrl), cls); + T validatedGet(String queryUrl, Map parameters, + Function deserializer) { + String logUri = "GET " + buildUri(queryUrl, parameters, false).toString(); + return retryableValidate(logUri, () -> { + URI queryUri = buildUri(queryUrl, parameters, true); + return get(queryUri, deserializer); + }); } - T post(String queryUrl, R body, Class requestCls, Class responseCls) { - return HttpHelper.doPost(appendToken(marketoEndpoint + queryUrl), body, requestCls, responseCls); + private T validatedPost(String queryUrl, Map parameters, + Function deserializer, + B body, Function qSerializer) { + String logUri = "POST " + buildUri(queryUrl, parameters, false).toString(); + return retryableValidate(logUri, () -> { + URI queryUri = buildUri(queryUrl, parameters, true); + return post(queryUri, deserializer, body, qSerializer); + }); } - String get(String queryUrl) { - return HttpHelper.doGet(queryUrl); + private T retryableValidate(String logUri, Supplier tryQuery) { + T result = tryQuery.get(); + // check for expired token + if (!result.isSuccess()) { + for (Error error : result.getErrors()) { + if (error.getCode() == 602 && error.getMessage().equals("Access token expired")) { + // refresh token and retry + token = refreshToken(); + LOG.info("Refreshed token"); + return tryQuery.get(); + } + } + } + + // log warnings if required + if (result.getWarnings().size() > 0) { + String warnings = result.getWarnings().stream() + .map(error -> String.format("code: %s, message: %s", error.getCode(), error.getMessage())) + .collect(Collectors.joining("; ")); + LOG.warn("Warnings when calling '{}' - {}", logUri, warnings); + } + + if (!result.isSuccess()) { + String msg = String.format("Errors when calling '%s'", logUri); + // log errors if required + if (result.getErrors().size() > 0) { + String errors = result.getErrors().stream() + .map(error -> String.format("code: %s, message: %s", error.getCode(), error.getMessage())) + .collect(Collectors.joining("; ")); + msg = msg + " - " + errors; + LOG.error(msg); + } + throw new RuntimeException(msg); + } + return result; } - public static MarketoToken getToken(String marketoEndpoint, String clientId, String clientSecret) { - LOG.info("Requesting marketo token"); - String marketoIdURL = marketoEndpoint + "/identity"; - String idEndpoint = marketoIdURL + "/oauth/token?grant_type=client_credentials&client_id=" - + clientId + "&client_secret=" + clientSecret; + T get(URI uri, Function deserializer) { + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + HttpGet request = new HttpGet(uri); + try (CloseableHttpResponse response = httpClient.execute(request, httpClientContext)) { + if(response.getStatusLine().getStatusCode() >= 300) { + throw new IOException(IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8)); + } + return deserializer.apply(response.getEntity().getContent()); + } + } catch (Exception e) { + throw Helpers.failForUri("GET", uri, e); + } + } - return HttpHelper.doGet(idEndpoint, MarketoToken.class); + private T post(URI uri, Function respDeserializer, B body, Function qSerializer) { + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + HttpPost request = new HttpPost(uri); + if (body != null) { + Objects.requireNonNull(qSerializer, "body serializer must be specified with body"); + request.setEntity(new StringEntity(qSerializer.apply(body), ContentType.APPLICATION_JSON)); + } + try (CloseableHttpResponse response = httpClient.execute(request, httpClientContext)) { + return respDeserializer.apply(response.getEntity().getContent()); + } + } catch (Exception e) { + throw Helpers.failForUri("POST", uri, e); + } } - private String appendToken(String requestUrl) { - if (!requestUrl.contains("access_token")) { - if (requestUrl.contains("?")) { - return requestUrl + "&access_token=" + token.getAccessToken(); - } else { - return requestUrl + "?access_token=" + token.getAccessToken(); + + URI buildUri(String queryUrl, Map parameters) { + return buildUri(queryUrl, parameters, true); + } + + URI buildUri(String queryUrl, Map parameters, boolean includeToken) { + try { + URIBuilder builder = new URIBuilder(marketoEndpoint + queryUrl); + parameters.forEach(builder::setParameter); + if (includeToken) { + builder.setParameter("access_token", token.getAccessToken()); } + return builder.build(); + } catch (URISyntaxException e) { + throw new IllegalArgumentException(String.format("'%s' is invalid URI", marketoEndpoint + queryUrl)); } - return requestUrl; } + MarketoToken getCurrentToken() { + return this.token; + } + + private MarketoToken refreshToken() { + LOG.debug("Requesting marketo token"); + URI getTokenUri = buildUri("/identity/oauth/token", + ImmutableMap.of("grant_type", "client_credentials", "client_id", clientId, + "client_secret", clientSecret), false); + return get(getTokenUri, inputStream -> GSON.fromJson(new InputStreamReader(inputStream), MarketoToken.class)); + } } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/MarketoPageIterator.java b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoPageIterator.java index fc1a0a9..3a7a424 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/MarketoPageIterator.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoPageIterator.java @@ -1,6 +1,5 @@ package io.cdap.plugin.marketo.common.api; -import com.google.gson.reflect.TypeToken; import io.cdap.plugin.marketo.common.api.entities.BaseResponse; import java.util.Iterator; @@ -18,16 +17,16 @@ public class MarketoPageIterator implements Iterator< private T currentPage; private Marketo marketo; private String queryUrl; - private TypeToken pageTypeToken; + private Class pageClass; private Function> resultsGetter; private Iterator currentPageResultIterator; - MarketoPageIterator(T page, Marketo marketo, String queryUrl, TypeToken pageTypeToken, + MarketoPageIterator(T page, Marketo marketo, String queryUrl, Class pageClass, Function> resultsGetter) { this.currentPage = page; this.marketo = marketo; this.queryUrl = queryUrl; - this.pageTypeToken = pageTypeToken; + this.pageClass = pageClass; this.resultsGetter = resultsGetter; currentPageResultIterator = resultsGetter.apply(this.currentPage).iterator(); } @@ -37,7 +36,7 @@ public boolean hasNext() { if (currentPageResultIterator.hasNext()) { return true; } else { - T nextPage = marketo.getNextPage(currentPage, queryUrl, pageTypeToken); + T nextPage = marketo.getNextPage(currentPage, queryUrl, pageClass); if (nextPage != null) { currentPage = nextPage; currentPageResultIterator = resultsGetter.apply(this.currentPage).iterator(); diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/BaseResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/BaseResponse.java index 368fe0e..dc3deb0 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/BaseResponse.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/BaseResponse.java @@ -10,7 +10,7 @@ public class BaseResponse { private boolean success = false; private List errors = Collections.emptyList(); - private List warnings = Collections.emptyList(); + private List warnings = Collections.emptyList(); private String requestId; private boolean moreResult = false; private String nextPageToken; @@ -19,23 +19,47 @@ public boolean isSuccess() { return success; } + public void setSuccess(boolean success) { + this.success = success; + } + public List getErrors() { return errors; } - public List getWarnings() { + public void setErrors(List errors) { + this.errors = errors; + } + + public List getWarnings() { return warnings; } + public void setWarnings(List warnings) { + this.warnings = warnings; + } + public String getRequestId() { return requestId; } + public void setRequestId(String requestId) { + this.requestId = requestId; + } + public boolean isMoreResult() { return moreResult; } + public void setMoreResult(boolean moreResult) { + this.moreResult = moreResult; + } + public String getNextPageToken() { return nextPageToken; } + + public void setNextPageToken(String nextPageToken) { + this.nextPageToken = nextPageToken; + } } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/Error.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/Error.java index 681577d..2c9a9e2 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/Error.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/Error.java @@ -7,6 +7,14 @@ public class Error { private int code; private String message; + public Error(int code, String message) { + this.code = code; + this.message = message; + } + + public Error() { + } + public int getCode() { return code; } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/Warning.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/Warning.java index 562ffa1..5722e51 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/Warning.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/Warning.java @@ -7,6 +7,14 @@ public class Warning { private int code; private String message; + public Warning(int code, String message) { + this.code = code; + this.message = message; + } + + public Warning() { + } + public int getCode() { return code; } diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java index f139ddf..c9cc414 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java @@ -115,7 +115,7 @@ public Schema getSchema() { public Marketo getMarketo() { if (marketo == null) { - marketo = new Marketo(getRestApiEndpoint(), getMarketoToken()); + marketo = new Marketo(getRestApiEndpoint(), getClientId(), getClientSecret()); } return marketo; } @@ -152,11 +152,4 @@ public String getStartDate() { public String getEndDate() { return endDate; } - - public MarketoToken getMarketoToken() { - if (token == null) { - token = Marketo.getToken(getRestApiEndpoint(), getClientId(), getClientSecret()); - } - return token; - } } diff --git a/src/test/java/io/cdap/plugin/marketo/common/api/MarketoTest.java b/src/test/java/io/cdap/plugin/marketo/common/api/MarketoTest.java new file mode 100644 index 0000000..04d2d3e --- /dev/null +++ b/src/test/java/io/cdap/plugin/marketo/common/api/MarketoTest.java @@ -0,0 +1,187 @@ +package io.cdap.plugin.marketo.common.api; + +import com.github.tomakehurst.wiremock.client.WireMock; +import com.github.tomakehurst.wiremock.core.WireMockConfiguration; +import com.github.tomakehurst.wiremock.junit.WireMockRule; +import com.github.tomakehurst.wiremock.stubbing.Scenario; +import com.google.common.collect.ImmutableMap; +import com.google.gson.Gson; +import io.cdap.plugin.marketo.common.api.entities.BaseResponse; +import io.cdap.plugin.marketo.common.api.entities.Error; +import io.cdap.plugin.marketo.common.api.entities.MarketoToken; +import io.cdap.plugin.marketo.common.api.entities.Warning; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +public class MarketoTest { + private static Gson GSON = new Gson(); + + @Rule + public WireMockRule wireMockRule = new WireMockRule( + WireMockConfiguration.wireMockConfig().dynamicPort() + ); + + public static class StubResponse extends BaseResponse { + StubResponse(boolean success, List errors, List warnings) { + setSuccess(success); + setErrors(errors); + setWarnings(warnings); + } + } + + @Test + public void testToken() { + WireMock.stubFor( + WireMock.get(WireMock.urlPathMatching("/identity/oauth/token")) + .withQueryParam("grant_type", WireMock.equalTo("client_credentials")) + .withQueryParam("client_id", WireMock.equalTo("clientNiceId")) + .withQueryParam("client_secret", WireMock.equalTo("clientNiceSecret")) + .willReturn( + WireMock.aResponse().withBody( + GSON.toJson(new MarketoToken("niceToken", "hello@world.com", "3600", "bearer") + ) + ) + ) + ); + + Marketo m = new Marketo(getApiUrl(), "clientNiceId", "clientNiceSecret"); + Assert.assertEquals("niceToken", m.getCurrentToken().getAccessToken()); + } + + @Test + public void testTokenRefresh() { + setupToken(); + + WireMock.stubFor( + WireMock.get(WireMock.urlPathMatching("/rest/v1/stub.json")).inScenario("retry") + .whenScenarioStateIs(Scenario.STARTED) + .willReturn( + WireMock.aResponse().withBody( + GSON.toJson(new StubResponse(false, + Collections.singletonList( + new Error(602, "Access token expired")), + Collections.emptyList())) + ) + ) + .willSetStateTo("refreshed") + ); + WireMock.stubFor( + WireMock.get(WireMock.urlPathMatching("/rest/v1/stub.json")).inScenario("retry") + .whenScenarioStateIs("refreshed") + .willReturn( + WireMock.aResponse().withBody( + GSON.toJson(new StubResponse(true, Collections.emptyList(), Collections.emptyList())) + ) + ) + ); + Marketo m = new Marketo(getApiUrl(), "clientNiceId", "clientNiceSecret"); + m.validatedGet("/rest/v1/stub.json", Collections.emptyMap(), + inputStream -> Helpers.streamToObject(inputStream, StubResponse.class)); + WireMock.verify(WireMock.exactly(2), + WireMock.getRequestedFor(WireMock.urlPathEqualTo("/identity/oauth/token"))); + WireMock.verify(WireMock.exactly(2), + WireMock.getRequestedFor(WireMock.urlPathEqualTo("/rest/v1/stub.json"))); + } + + @Test + public void testMessages() { + setupToken(); + + WireMock.stubFor( + WireMock.get(WireMock.urlPathMatching("/rest/v1/justWarnings.json")) + .willReturn( + WireMock.aResponse().withBody( + GSON.toJson(new StubResponse(true, Collections.emptyList(), Arrays.asList( + new Warning(700, "Reversed agent 007"), + new Warning(777, "Result of 1000 - 333") + )))) + ) + ); + + WireMock.stubFor( + WireMock.get(WireMock.urlPathMatching("/rest/v1/errors.json")) + .willReturn( + WireMock.aResponse().withBody( + GSON.toJson(new StubResponse(false, + Collections.singletonList(new Error(123, "No way")), + Collections.emptyList()))) + ) + ); + + Marketo m = new Marketo(getApiUrl(), "clientNiceId", "clientNiceSecret"); + m.validatedGet("/rest/v1/justWarnings.json", Collections.emptyMap(), + inputStream -> Helpers.streamToObject(inputStream, StubResponse.class)); + + try { + m.validatedGet("/rest/v1/errors.json", Collections.emptyMap(), + inputStream -> Helpers.streamToObject(inputStream, StubResponse.class)); + Assert.fail("This call expected to fail."); + } catch (RuntimeException ex) { + Assert.assertTrue(ex.getMessage().contains("123")); + Assert.assertTrue(ex.getMessage().contains("No way")); + } + } + + @Test + public void testBuildUri() { + setupToken(); + Marketo m = new Marketo(getApiUrl(), "clientNiceId", "clientNiceSecret"); + String uriWithToken = m.buildUri("/hello", Collections.emptyMap()).toString(); + Assert.assertTrue(uriWithToken.contains("access_token")); + String uriWithoutToken = m.buildUri("/hello", ImmutableMap.of("param", "value"), false).toString(); + Assert.assertFalse(uriWithoutToken.contains("access_token")); + Assert.assertTrue(uriWithoutToken.contains("param=value")); + } + + @Test + public void testHttpError() { + setupToken(); + + WireMock.stubFor( + WireMock.get(WireMock.urlPathMatching("/rest/v1/fail.json")) + .willReturn( + WireMock.aResponse().withStatus(500).withBody("GJ server.") + ) + ); + + try { + Marketo m = new Marketo(getApiUrl(), "clientNiceId", "clientNiceSecret"); + m.validatedGet("/rest/v1/fail.json", Collections.emptyMap(), + inputStream -> Helpers.streamToObject(inputStream, StubResponse.class)); + Assert.fail("This call expected to fail."); + } catch (RuntimeException ex) { + Assert.assertTrue(ex.getMessage().contains("GJ server")); + } + } + + @Test + public void invalidEndpoint() { + try { + new Marketo("%^%^&%^", "clientNiceId", "clientNiceSecret"); + Assert.fail("This call expected to fail."); + } catch (IllegalArgumentException ex) { + Assert.assertEquals("'%^%^&%^/identity/oauth/token' is invalid URI", ex.getMessage()); + } + } + + void setupToken() { + WireMock.stubFor( + WireMock.get(WireMock.urlPathMatching("/identity/oauth/token")) + .willReturn( + WireMock.aResponse().withBody( + GSON.toJson(new MarketoToken("niceToken", "hello@world.com", "3600", "bearer") + ) + ) + ) + ); + } + + String getApiUrl() { + return String.format("http://localhost:%d", wireMockRule.port()); + } +} \ No newline at end of file From 2cd7bd3a6548db928a54906773cf437fcd2dfc90 Mon Sep 17 00:00:00 2001 From: Yevhenii Chekanskyi Date: Tue, 26 Nov 2019 01:37:05 +0200 Subject: [PATCH 04/12] PLUGIN-75 Marketo Plugin - add some tests. --- pom.xml | 12 +- .../plugin/marketo/common/api/Helpers.java | 3 + .../marketo/common/api/LeadsExportJob.java | 5 +- .../plugin/marketo/common/api/Marketo.java | 179 +---------------- .../marketo/common/api/MarketoHttp.java | 188 ++++++++++++++++++ .../common/api/MarketoPageIterator.java | 4 +- ...{MarketoTest.java => MarketoHttpTest.java} | 106 +++++++++- 7 files changed, 302 insertions(+), 195 deletions(-) create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java rename src/test/java/io/cdap/plugin/marketo/common/api/{MarketoTest.java => MarketoHttpTest.java} (63%) diff --git a/pom.xml b/pom.xml index d237f88..b697a2b 100644 --- a/pom.xml +++ b/pom.xml @@ -260,12 +260,12 @@ 2.6 - - org.slf4j - slf4j-simple - 1.7.29 - - + + + + + + junit junit diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java b/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java index 78c77ad..7d109cd 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java @@ -13,6 +13,9 @@ import java.nio.charset.StandardCharsets; import java.util.List; +/** + * Various helper methods. + */ public class Helpers { public static String streamToString(InputStream inputStream) { try { diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java b/src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java index 2feaf0f..8e53e9c 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java @@ -54,8 +54,9 @@ public void waitCompletion() throws InterruptedException { } public void enqueue() { - last = marketo.post(String.format(Urls.BULK_EXPORT_LEADS_ENQUEUE, jobId), - null, LeadsExport.class).singleExport(); + last = marketo.validatedPost(String.format(Urls.BULK_EXPORT_LEADS_ENQUEUE, jobId), Collections.emptyMap(), + inputStream -> Helpers.streamToObject(inputStream, LeadsExport.class), + null, null).singleExport(); LOG.info("Bulk lead export job with id '{}' enqueued", jobId); } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java b/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java index 49e569b..8b6fda4 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java @@ -16,62 +16,29 @@ package io.cdap.plugin.marketo.common.api; -import com.google.common.base.Strings; -import com.google.common.collect.ImmutableMap; import com.google.gson.Gson; -import io.cdap.plugin.marketo.common.api.entities.BaseResponse; -import io.cdap.plugin.marketo.common.api.entities.Error; -import io.cdap.plugin.marketo.common.api.entities.MarketoToken; import io.cdap.plugin.marketo.common.api.entities.leads.LeadsDescribe; import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExport; import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExportRequest; -import org.apache.commons.io.IOUtils; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.protocol.HttpClientContext; -import org.apache.http.client.utils.URIBuilder; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.List; -import java.util.Map; -import java.util.Objects; import java.util.Spliterator; import java.util.Spliterators; -import java.util.function.Function; -import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.StreamSupport; /** - * Helper class to perform Marketo Api operations. + * Class that expose marketo rest api endpoints. */ -public class Marketo { +public class Marketo extends MarketoHttp { private static final Logger LOG = LoggerFactory.getLogger(Marketo.class); static final Gson GSON = new Gson(); - private String marketoEndpoint; - private String clientId; - private String clientSecret; - private MarketoToken token; - private HttpClientContext httpClientContext = HttpClientContext.create(); public Marketo(String marketoEndpoint, String clientId, String clientSecret) { - this.marketoEndpoint = marketoEndpoint; - this.clientId = clientId; - this.clientSecret = clientSecret; - token = refreshToken(); + super(marketoEndpoint, clientId, clientSecret); } public List describeLeads() { @@ -88,144 +55,4 @@ public LeadsExportJob exportLeads(LeadsExportRequest request) { return new LeadsExportJob(export, this); } - private T getPage(String queryUrl, Class pageClass) { - return validatedGet(queryUrl, Collections.emptyMap(), - inputStream -> Helpers.streamToObject(inputStream, pageClass)); - } - - T getNextPage(T currentPage, String queryUrl, Class pageClass) { - if (!Strings.isNullOrEmpty(currentPage.getNextPageToken())) { - return validatedGet(queryUrl, - ImmutableMap.of("nextPageToken", currentPage.getNextPageToken()), - inputStream -> Helpers.streamToObject(inputStream, pageClass)); - } - return null; - } - - private MarketoPageIterator iteratePage(String queryUrl, - Class pageClass, - Function> resultsGetter) { - return new MarketoPageIterator<>(getPage(queryUrl, pageClass), this, queryUrl, pageClass, resultsGetter); - } - - T post(String queryUrl, R body, Class responseCls) { - return validatedPost(queryUrl, Collections.emptyMap(), - inputStream -> Helpers.streamToObject(inputStream, responseCls), - body, GSON::toJson); - } - - T validatedGet(String queryUrl, Map parameters, - Function deserializer) { - String logUri = "GET " + buildUri(queryUrl, parameters, false).toString(); - return retryableValidate(logUri, () -> { - URI queryUri = buildUri(queryUrl, parameters, true); - return get(queryUri, deserializer); - }); - } - - private T validatedPost(String queryUrl, Map parameters, - Function deserializer, - B body, Function qSerializer) { - String logUri = "POST " + buildUri(queryUrl, parameters, false).toString(); - return retryableValidate(logUri, () -> { - URI queryUri = buildUri(queryUrl, parameters, true); - return post(queryUri, deserializer, body, qSerializer); - }); - } - - private T retryableValidate(String logUri, Supplier tryQuery) { - T result = tryQuery.get(); - // check for expired token - if (!result.isSuccess()) { - for (Error error : result.getErrors()) { - if (error.getCode() == 602 && error.getMessage().equals("Access token expired")) { - // refresh token and retry - token = refreshToken(); - LOG.info("Refreshed token"); - return tryQuery.get(); - } - } - } - - // log warnings if required - if (result.getWarnings().size() > 0) { - String warnings = result.getWarnings().stream() - .map(error -> String.format("code: %s, message: %s", error.getCode(), error.getMessage())) - .collect(Collectors.joining("; ")); - LOG.warn("Warnings when calling '{}' - {}", logUri, warnings); - } - - if (!result.isSuccess()) { - String msg = String.format("Errors when calling '%s'", logUri); - // log errors if required - if (result.getErrors().size() > 0) { - String errors = result.getErrors().stream() - .map(error -> String.format("code: %s, message: %s", error.getCode(), error.getMessage())) - .collect(Collectors.joining("; ")); - msg = msg + " - " + errors; - LOG.error(msg); - } - throw new RuntimeException(msg); - } - return result; - } - - T get(URI uri, Function deserializer) { - try (CloseableHttpClient httpClient = HttpClients.createDefault()) { - HttpGet request = new HttpGet(uri); - try (CloseableHttpResponse response = httpClient.execute(request, httpClientContext)) { - if(response.getStatusLine().getStatusCode() >= 300) { - throw new IOException(IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8)); - } - return deserializer.apply(response.getEntity().getContent()); - } - } catch (Exception e) { - throw Helpers.failForUri("GET", uri, e); - } - } - - private T post(URI uri, Function respDeserializer, B body, Function qSerializer) { - try (CloseableHttpClient httpClient = HttpClients.createDefault()) { - HttpPost request = new HttpPost(uri); - if (body != null) { - Objects.requireNonNull(qSerializer, "body serializer must be specified with body"); - request.setEntity(new StringEntity(qSerializer.apply(body), ContentType.APPLICATION_JSON)); - } - try (CloseableHttpResponse response = httpClient.execute(request, httpClientContext)) { - return respDeserializer.apply(response.getEntity().getContent()); - } - } catch (Exception e) { - throw Helpers.failForUri("POST", uri, e); - } - } - - - URI buildUri(String queryUrl, Map parameters) { - return buildUri(queryUrl, parameters, true); - } - - URI buildUri(String queryUrl, Map parameters, boolean includeToken) { - try { - URIBuilder builder = new URIBuilder(marketoEndpoint + queryUrl); - parameters.forEach(builder::setParameter); - if (includeToken) { - builder.setParameter("access_token", token.getAccessToken()); - } - return builder.build(); - } catch (URISyntaxException e) { - throw new IllegalArgumentException(String.format("'%s' is invalid URI", marketoEndpoint + queryUrl)); - } - } - - MarketoToken getCurrentToken() { - return this.token; - } - - private MarketoToken refreshToken() { - LOG.debug("Requesting marketo token"); - URI getTokenUri = buildUri("/identity/oauth/token", - ImmutableMap.of("grant_type", "client_credentials", "client_id", clientId, - "client_secret", clientSecret), false); - return get(getTokenUri, inputStream -> GSON.fromJson(new InputStreamReader(inputStream), MarketoToken.class)); - } } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java new file mode 100644 index 0000000..f71cd77 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java @@ -0,0 +1,188 @@ +package io.cdap.plugin.marketo.common.api; + +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; +import com.google.gson.Gson; +import io.cdap.plugin.marketo.common.api.entities.BaseResponse; +import io.cdap.plugin.marketo.common.api.entities.Error; +import io.cdap.plugin.marketo.common.api.entities.MarketoToken; +import org.apache.commons.io.IOUtils; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +/** + * Class that encapsulates common http functions for marketo rest api. + */ +class MarketoHttp { + private static final Logger LOG = LoggerFactory.getLogger(Marketo.class); + private static final Gson GSON = new Gson(); + private String marketoEndpoint; + private String clientId; + private String clientSecret; + private MarketoToken token; + private HttpClientContext httpClientContext = HttpClientContext.create(); + + MarketoHttp(String marketoEndpoint, String clientId, String clientSecret) { + this.marketoEndpoint = marketoEndpoint; + this.clientId = clientId; + this.clientSecret = clientSecret; + token = refreshToken(); + } + + private T getPage(String queryUrl, Class pageClass) { + return validatedGet(queryUrl, Collections.emptyMap(), + inputStream -> Helpers.streamToObject(inputStream, pageClass)); + } + + T getNextPage(T currentPage, String queryUrl, Class pageClass) { + if (!Strings.isNullOrEmpty(currentPage.getNextPageToken())) { + return validatedGet(queryUrl, + ImmutableMap.of("nextPageToken", currentPage.getNextPageToken()), + inputStream -> Helpers.streamToObject(inputStream, pageClass)); + } + return null; + } + + MarketoPageIterator iteratePage(String queryUrl, + Class pageClass, + Function> resultsGetter) { + return new MarketoPageIterator<>(getPage(queryUrl, pageClass), this, queryUrl, pageClass, resultsGetter); + } + + T validatedGet(String queryUrl, Map parameters, + Function deserializer) { + String logUri = "GET " + buildUri(queryUrl, parameters, false).toString(); + return retryableValidate(logUri, () -> { + URI queryUri = buildUri(queryUrl, parameters, true); + return get(queryUri, deserializer); + }); + } + + T validatedPost(String queryUrl, Map parameters, + Function deserializer, + B body, Function qSerializer) { + String logUri = "POST " + buildUri(queryUrl, parameters, false).toString(); + return retryableValidate(logUri, () -> { + URI queryUri = buildUri(queryUrl, parameters, true); + return post(queryUri, deserializer, body, qSerializer); + }); + } + + private T retryableValidate(String logUri, Supplier tryQuery) { + T result = tryQuery.get(); + // check for expired token + if (!result.isSuccess()) { + for (Error error : result.getErrors()) { + if (error.getCode() == 602 && error.getMessage().equals("Access token expired")) { + // refresh token and retry + token = refreshToken(); + LOG.info("Refreshed token"); + return tryQuery.get(); + } + } + } + + // log warnings if required + if (result.getWarnings().size() > 0) { + String warnings = result.getWarnings().stream() + .map(error -> String.format("code: %s, message: %s", error.getCode(), error.getMessage())) + .collect(Collectors.joining("; ")); + LOG.warn("Warnings when calling '{}' - {}", logUri, warnings); + } + + if (!result.isSuccess()) { + String msg = String.format("Errors when calling '%s'", logUri); + // log errors if required + if (result.getErrors().size() > 0) { + String errors = result.getErrors().stream() + .map(error -> String.format("code: %s, message: %s", error.getCode(), error.getMessage())) + .collect(Collectors.joining("; ")); + msg = msg + " - " + errors; + LOG.error(msg); + } + throw new RuntimeException(msg); + } + return result; + } + + T get(URI uri, Function deserializer) { + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + HttpGet request = new HttpGet(uri); + try (CloseableHttpResponse response = httpClient.execute(request, httpClientContext)) { + if (response.getStatusLine().getStatusCode() >= 300) { + throw new IOException(IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8)); + } + return deserializer.apply(response.getEntity().getContent()); + } + } catch (Exception e) { + throw Helpers.failForUri("GET", uri, e); + } + } + + private T post(URI uri, Function respDeserializer, B body, Function qSerializer) { + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + HttpPost request = new HttpPost(uri); + if (body != null) { + Objects.requireNonNull(qSerializer, "body serializer must be specified with body"); + request.setEntity(new StringEntity(qSerializer.apply(body), ContentType.APPLICATION_JSON)); + } + try (CloseableHttpResponse response = httpClient.execute(request, httpClientContext)) { + return respDeserializer.apply(response.getEntity().getContent()); + } + } catch (Exception e) { + throw Helpers.failForUri("POST", uri, e); + } + } + + URI buildUri(String queryUrl, Map parameters) { + return buildUri(queryUrl, parameters, true); + } + + URI buildUri(String queryUrl, Map parameters, boolean includeToken) { + try { + URIBuilder builder = new URIBuilder(marketoEndpoint + queryUrl); + parameters.forEach(builder::setParameter); + if (includeToken) { + builder.setParameter("access_token", token.getAccessToken()); + } + return builder.build(); + } catch (URISyntaxException e) { + throw new IllegalArgumentException(String.format("'%s' is invalid URI", marketoEndpoint + queryUrl)); + } + } + + MarketoToken getCurrentToken() { + return this.token; + } + + private MarketoToken refreshToken() { + LOG.debug("Requesting marketo token"); + URI getTokenUri = buildUri("/identity/oauth/token", + ImmutableMap.of("grant_type", "client_credentials", "client_id", clientId, + "client_secret", clientSecret), false); + return get(getTokenUri, inputStream -> GSON.fromJson(new InputStreamReader(inputStream), MarketoToken.class)); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/MarketoPageIterator.java b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoPageIterator.java index 3a7a424..0c32729 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/MarketoPageIterator.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoPageIterator.java @@ -15,13 +15,13 @@ */ public class MarketoPageIterator implements Iterator { private T currentPage; - private Marketo marketo; + private MarketoHttp marketo; private String queryUrl; private Class pageClass; private Function> resultsGetter; private Iterator currentPageResultIterator; - MarketoPageIterator(T page, Marketo marketo, String queryUrl, Class pageClass, + MarketoPageIterator(T page, MarketoHttp marketo, String queryUrl, Class pageClass, Function> resultsGetter) { this.currentPage = page; this.marketo = marketo; diff --git a/src/test/java/io/cdap/plugin/marketo/common/api/MarketoTest.java b/src/test/java/io/cdap/plugin/marketo/common/api/MarketoHttpTest.java similarity index 63% rename from src/test/java/io/cdap/plugin/marketo/common/api/MarketoTest.java rename to src/test/java/io/cdap/plugin/marketo/common/api/MarketoHttpTest.java index 04d2d3e..3ad8844 100644 --- a/src/test/java/io/cdap/plugin/marketo/common/api/MarketoTest.java +++ b/src/test/java/io/cdap/plugin/marketo/common/api/MarketoHttpTest.java @@ -17,9 +17,13 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; -public class MarketoTest { - private static Gson GSON = new Gson(); +public class MarketoHttpTest { + private static final Gson GSON = new Gson(); @Rule public WireMockRule wireMockRule = new WireMockRule( @@ -34,6 +38,67 @@ public static class StubResponse extends BaseResponse { } } + public static class PageResponse extends BaseResponse { + private List results; + + PageResponse(boolean moreResults, String nextPageToken, String... items) { + setSuccess(true); + setMoreResult(moreResults); + results = Arrays.asList(items); + setNextPageToken(nextPageToken); + } + + public List getResults() { + return results; + } + } + + @Test + public void testPaging() { + setupToken(); + + WireMock.stubFor( + WireMock.get(WireMock.urlPathMatching("/rest/v1/paged.json")).inScenario("page") + .whenScenarioStateIs(Scenario.STARTED) + .willReturn( + WireMock.aResponse().withBody( + GSON.toJson(new PageResponse(true, "page1", "1", "2", "3")) + ) + ) + .willSetStateTo("page1") + ); + WireMock.stubFor( + WireMock.get(WireMock.urlPathMatching("/rest/v1/paged.json")).inScenario("page") + .whenScenarioStateIs("page1") + .withQueryParam("nextPageToken", WireMock.equalTo("page1")) + .willReturn( + WireMock.aResponse().withBody( + GSON.toJson(new PageResponse(true, "page2", "4", "5", "6")) + ) + ) + .willSetStateTo("page2") + ); + WireMock.stubFor( + WireMock.get(WireMock.urlPathMatching("/rest/v1/paged.json")).inScenario("page") + .whenScenarioStateIs("page2") + .withQueryParam("nextPageToken", WireMock.equalTo("page2")) + .willReturn( + WireMock.aResponse().withBody( + GSON.toJson(new PageResponse(false, null, "7", "8", "9")) + ) + ) + .willSetStateTo("page3") + ); + + MarketoHttp m = new MarketoHttp(getApiUrl(), "clientNiceId", "clientNiceSecret"); + + List results = StreamSupport.stream(Spliterators.spliteratorUnknownSize( + m.iteratePage("/rest/v1/paged.json", PageResponse.class, PageResponse::getResults), + Spliterator.ORDERED), false).sorted().collect(Collectors.toList()); + + Assert.assertArrayEquals(new String[]{"1", "2", "3", "4", "5", "6", "7", "8", "9"}, results.toArray()); + } + @Test public void testToken() { WireMock.stubFor( @@ -49,7 +114,7 @@ public void testToken() { ) ); - Marketo m = new Marketo(getApiUrl(), "clientNiceId", "clientNiceSecret"); + MarketoHttp m = new MarketoHttp(getApiUrl(), "clientNiceId", "clientNiceSecret"); Assert.assertEquals("niceToken", m.getCurrentToken().getAccessToken()); } @@ -79,7 +144,7 @@ public void testTokenRefresh() { ) ) ); - Marketo m = new Marketo(getApiUrl(), "clientNiceId", "clientNiceSecret"); + MarketoHttp m = new MarketoHttp(getApiUrl(), "clientNiceId", "clientNiceSecret"); m.validatedGet("/rest/v1/stub.json", Collections.emptyMap(), inputStream -> Helpers.streamToObject(inputStream, StubResponse.class)); WireMock.verify(WireMock.exactly(2), @@ -88,6 +153,29 @@ public void testTokenRefresh() { WireMock.getRequestedFor(WireMock.urlPathEqualTo("/rest/v1/stub.json"))); } + @Test + public void testPost() { + setupToken(); + + WireMock.stubFor( + WireMock.post(WireMock.urlPathMatching("/rest/v1/post.json")) + .willReturn( + WireMock.aResponse().withBody( + GSON.toJson(new StubResponse(true, Collections.emptyList(), Collections.emptyList())) + ) + ) + ); + + MarketoHttp m = new MarketoHttp(getApiUrl(), "clientNiceId", "clientNiceSecret"); + m.validatedPost("/rest/v1/post.json", Collections.emptyMap(), + inputStream -> Helpers.streamToObject(inputStream, StubResponse.class), "body", String::toString); + + WireMock.verify( + WireMock.postRequestedFor(WireMock.urlPathEqualTo("/rest/v1/post.json")) + .withRequestBody(WireMock.equalTo("body")) + ); + } + @Test public void testMessages() { setupToken(); @@ -113,7 +201,7 @@ public void testMessages() { ) ); - Marketo m = new Marketo(getApiUrl(), "clientNiceId", "clientNiceSecret"); + MarketoHttp m = new MarketoHttp(getApiUrl(), "clientNiceId", "clientNiceSecret"); m.validatedGet("/rest/v1/justWarnings.json", Collections.emptyMap(), inputStream -> Helpers.streamToObject(inputStream, StubResponse.class)); @@ -130,7 +218,7 @@ public void testMessages() { @Test public void testBuildUri() { setupToken(); - Marketo m = new Marketo(getApiUrl(), "clientNiceId", "clientNiceSecret"); + MarketoHttp m = new MarketoHttp(getApiUrl(), "clientNiceId", "clientNiceSecret"); String uriWithToken = m.buildUri("/hello", Collections.emptyMap()).toString(); Assert.assertTrue(uriWithToken.contains("access_token")); String uriWithoutToken = m.buildUri("/hello", ImmutableMap.of("param", "value"), false).toString(); @@ -150,7 +238,7 @@ public void testHttpError() { ); try { - Marketo m = new Marketo(getApiUrl(), "clientNiceId", "clientNiceSecret"); + MarketoHttp m = new MarketoHttp(getApiUrl(), "clientNiceId", "clientNiceSecret"); m.validatedGet("/rest/v1/fail.json", Collections.emptyMap(), inputStream -> Helpers.streamToObject(inputStream, StubResponse.class)); Assert.fail("This call expected to fail."); @@ -162,7 +250,7 @@ public void testHttpError() { @Test public void invalidEndpoint() { try { - new Marketo("%^%^&%^", "clientNiceId", "clientNiceSecret"); + new MarketoHttp("%^%^&%^", "clientNiceId", "clientNiceSecret"); Assert.fail("This call expected to fail."); } catch (IllegalArgumentException ex) { Assert.assertEquals("'%^%^&%^/identity/oauth/token' is invalid URI", ex.getMessage()); @@ -184,4 +272,4 @@ void setupToken() { String getApiUrl() { return String.format("http://localhost:%d", wireMockRule.port()); } -} \ No newline at end of file +} From 91ac74d58aa885ee3d85d16eaf41c1e3b1b63507 Mon Sep 17 00:00:00 2001 From: Yevhenii Chekanskyi Date: Tue, 26 Nov 2019 23:00:40 +0200 Subject: [PATCH 05/12] PLUGIN-75 Marketo Plugin - support splits. --- .../plugin/marketo/common/api/Helpers.java | 47 ++++++++++++ .../marketo/common/api/LeadsExportJob.java | 76 ++++++++++++------- .../plugin/marketo/common/api/Marketo.java | 45 ++++++++++- .../cdap/plugin/marketo/common/api/Urls.java | 1 + .../common/api/entities/DateRange.java | 31 ++++++++ .../api/entities/leads/LeadsExport.java | 4 + .../entities/leads/LeadsExportRequest.java | 15 +--- .../source/batch/MarketoInputFormat.java | 18 ++++- .../source/batch/MarketoRecordReader.java | 21 ++++- ...oSplit.java => MarketoReportingSplit.java} | 29 ++++++- 10 files changed, 234 insertions(+), 53 deletions(-) create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/DateRange.java rename src/main/java/io/cdap/plugin/marketo/source/batch/{NoOpMarketoSplit.java => MarketoReportingSplit.java} (58%) diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java b/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java index 7d109cd..7301043 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java @@ -1,6 +1,8 @@ package io.cdap.plugin.marketo.common.api; import com.google.common.base.Strings; +import com.google.common.collect.ImmutableList; +import io.cdap.plugin.marketo.common.api.entities.DateRange; import org.apache.commons.io.IOUtils; import org.apache.http.NameValuePair; import org.apache.http.client.utils.URIBuilder; @@ -11,6 +13,8 @@ import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; +import java.time.OffsetDateTime; +import java.util.ArrayList; import java.util.List; /** @@ -51,4 +55,47 @@ public static RuntimeException failForUri(String method, URI uri, Exception ex) return new RuntimeException(message); } } + + /** + * Splits date range in 30 day ranges. + * Return date range as is if difference is less or equals to 30 days. + * + * @param beginDate + * @param endDate + * @return + */ + public static List getDateRanges(String beginDate, String endDate) { + OffsetDateTime start = OffsetDateTime.parse(beginDate); + OffsetDateTime end = OffsetDateTime.parse(endDate); + + if (start.compareTo(end) > 0) { + throw new IllegalArgumentException("start date more than end date"); + } + + int compareResult = start.plusDays(30).compareTo(end); + if (compareResult >= 0) { + // we are in range of 30 days, dates are okay + return ImmutableList.of(new DateRange(start.toString(), end.toString())); + } else { + List result = new ArrayList<>(); + OffsetDateTime currentStart = start; + + while (currentStart.compareTo(end) < 0) { + OffsetDateTime nextEnd = currentStart.plusDays(30); + result.add(new DateRange(currentStart.toString(), + min(nextEnd.minusSeconds(1), end).toString())); + currentStart = nextEnd; + } + + return result; + } + } + + public static > T min(T o1, T o2) { + if (o1.compareTo(o2) < 0) { + return o1; + } else { + return o2; + } + } } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java b/src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java index 8e53e9c..9d2c13b 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java @@ -7,62 +7,82 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.concurrent.TimeUnit; /** * Leads export job. */ public class LeadsExportJob { private static final Logger LOG = LoggerFactory.getLogger(LeadsExportJob.class); - private static final List WAITABLE_STATE = Arrays.asList("Queued", "Processing"); - private static final List COMPLETED_STATUS = Arrays.asList("Canceled", "Completed", "Failed"); + private static final List WAIT_ABLE_STATE = Arrays.asList("Queued", "Processing"); + private static final String ENQUEUE_ABLE_STATUS = "Created"; + private static final String COMPLETED_STATUS = "Completed"; private String jobId; - private LeadsExport.ExportResponse last; + private LeadsExport.ExportResponse lastStatus; private Marketo marketo; - public LeadsExportJob(LeadsExport lastStatus, Marketo marketo) { - this.jobId = lastStatus.singleExport().getExportId(); - this.last = lastStatus.singleExport(); + public LeadsExportJob(LeadsExport.ExportResponse lastStatus, Marketo marketo) { + this.jobId = lastStatus.getExportId(); + this.lastStatus = lastStatus; this.marketo = marketo; - LOG.info("Created bulk lead export job with id '{}'", this.jobId); + LOG.info("BULK LEADS EXPORT - created job '{}'", this.jobId); } - public String getStatus() { - return last.getStatus(); + public String getLastStatus() { + return lastStatus.getStatus(); } public void waitCompletion() throws InterruptedException { - if (!WAITABLE_STATE.contains(getStatus())) { + if (!WAIT_ABLE_STATE.contains(getLastStatus())) { throw new IllegalStateException("Job must be enqueued before waiting for completion."); } - while (!COMPLETED_STATUS.contains(getStatus())) { - LeadsExport currentResp = marketo.validatedGet( - String.format(Urls.BULK_EXPORT_LEADS_STATUS, jobId), - Collections.emptyMap(), inputStream -> Helpers.streamToObject(inputStream, LeadsExport.class)); - LeadsExport.ExportResponse current = currentResp.singleExport(); - String previousStatus = getStatus(); - String currentStatus = current.getStatus(); - if (!currentStatus.equals(previousStatus)) { - LOG.info("Bulk lead export job with id '{}' changed status from '{}' to '{}'", jobId, previousStatus, - currentStatus); - } - last = current; - Thread.sleep(30 * 1000); + do { + Thread.sleep(TimeUnit.SECONDS.toMillis(30)); + LeadsExport.ExportResponse newState = marketo.leadsExportJobStatus(jobId); + logStatusChange(getLastStatus(), newState.getStatus()); + lastStatus = newState; + } while (WAIT_ABLE_STATE.contains(getLastStatus())); + + if (!getLastStatus().equals(COMPLETED_STATUS)) { + throw new IllegalStateException("Job expected to be in Completed state, but was in " + getLastStatus()); } - LOG.info("Bulk lead export job with id '{}' finished with status '{}'", jobId, getStatus()); } public void enqueue() { - last = marketo.validatedPost(String.format(Urls.BULK_EXPORT_LEADS_ENQUEUE, jobId), Collections.emptyMap(), - inputStream -> Helpers.streamToObject(inputStream, LeadsExport.class), - null, null).singleExport(); + if (!getLastStatus().equals(ENQUEUE_ABLE_STATUS)) { + throw new IllegalStateException("Job must be in Created status before enqueuing, but was in " + getLastStatus()); + } + + LeadsExport.ExportResponse newState = marketo.validatedPost( + String.format(Urls.BULK_EXPORT_LEADS_ENQUEUE, jobId), + Collections.emptyMap(), + inputStream -> Helpers.streamToObject(inputStream, LeadsExport.class), + null, null).singleExport(); + + logStatusChange(getLastStatus(), newState.getStatus()); + + if (!(newState.getStatus().equals("Queued") || newState.getStatus().equals("Processing"))) { + throw new IllegalStateException( + String.format("Expected Queued|Processing state for job '%s' but got '%s'", jobId, newState.getStatus())); + } - LOG.info("Bulk lead export job with id '{}' enqueued", jobId); + lastStatus = newState; + } + + private void logStatusChange(String oldStatus, String newStatus) { + if (!oldStatus.equals(newStatus)) { + LOG.info("BULK LEADS EXPORT - job '{}' changed state '{}' -> '{}'", jobId, oldStatus, newStatus); + } } public String getFile() { return marketo.get(marketo.buildUri(String.format(Urls.BULK_EXPORT_LEADS_FILE, jobId), Collections.emptyMap()), Helpers::streamToString); } + + public String getJobId() { + return jobId; + } } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java b/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java index 8b6fda4..d2bb9f9 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java @@ -16,6 +16,7 @@ package io.cdap.plugin.marketo.common.api; +import com.google.common.collect.ImmutableMap; import com.google.gson.Gson; import io.cdap.plugin.marketo.common.api.entities.leads.LeadsDescribe; import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExport; @@ -27,6 +28,7 @@ import java.util.List; import java.util.Spliterator; import java.util.Spliterators; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -52,7 +54,48 @@ public LeadsExportJob exportLeads(LeadsExportRequest request) { inputStream -> Helpers.streamToObject(inputStream, LeadsExport.class), request, GSON::toJson); - return new LeadsExportJob(export, this); + return new LeadsExportJob(export.singleExport(), this); } + public LeadsExport.ExportResponse leadsExportJobStatus(String jobId) { + LeadsExport currentResp = validatedGet( + String.format(Urls.BULK_EXPORT_LEADS_STATUS, jobId), + Collections.emptyMap(), inputStream -> Helpers.streamToObject(inputStream, LeadsExport.class)); + return currentResp.singleExport(); + } + + /** + * Waits until bulk extract queue has available slot and executes given action. + * + * @param action action to execute once slot is available + * @param timeoutSeconds timeout, in seconds + */ + public void onBulkExtractQueueAvailable(Runnable action, long timeoutSeconds) { + long timeoutMillis = TimeUnit.SECONDS.toMillis(timeoutSeconds); + long startTime = System.currentTimeMillis(); + while (System.currentTimeMillis() - startTime < timeoutMillis) { + if (canEnqueueJob()) { + action.run(); + return; + } else { + try { + Thread.sleep(TimeUnit.SECONDS.toMillis(60)); + } catch (InterruptedException e) { + throw new RuntimeException("Failed to get slot in bulk export queue - interrupted"); + } + } + } + throw new RuntimeException("Failed to get slot in bulk export queue - timeout"); + } + + private boolean canEnqueueJob() { + LeadsExport exportJobs = validatedGet(Urls.BULK_EXPORT_LEADS_LIST, + ImmutableMap.of("status", "queued,processing"), + inputStream -> Helpers.streamToObject(inputStream, LeadsExport.class) + ); + int jobsInQueue = exportJobs.getResult().size(); + LOG.debug("Jobs in queue: {}", jobsInQueue); + // TODO handle activity queue here + return jobsInQueue < 10; + } } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Urls.java b/src/main/java/io/cdap/plugin/marketo/common/api/Urls.java index dc0ae5f..2db9085 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/Urls.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Urls.java @@ -4,6 +4,7 @@ * Marketo API urls. */ public class Urls { + public static final String BULK_EXPORT_LEADS_LIST = "/bulk/v1/leads/export.json"; public static final String BULK_EXPORT_LEADS_CREATE = "/bulk/v1/leads/export/create.json"; public static final String BULK_EXPORT_LEADS_ENQUEUE = "/bulk/v1/leads/export/%s/enqueue.json"; public static final String BULK_EXPORT_LEADS_STATUS = "/bulk/v1/leads/export/%s/status.json"; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/DateRange.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/DateRange.java new file mode 100644 index 0000000..3764947 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/DateRange.java @@ -0,0 +1,31 @@ +package io.cdap.plugin.marketo.common.api.entities; + +/** + * Represents date range. + */ +public class DateRange { + String endAt; + String startAt; + + public DateRange() { + + } + + public DateRange(String startAt, String endAt) { + this.endAt = endAt; + this.startAt = startAt; + } + + public String getEndAt() { + return endAt; + } + + public String getStartAt() { + return startAt; + } + + @Override + public String toString() { + return startAt + " -- " + endAt; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExport.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExport.java index 877630b..65f6312 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExport.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExport.java @@ -79,4 +79,8 @@ public ExportResponse singleExport() { return result.get(0); } + public List getResult() { + return result; + } + } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportRequest.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportRequest.java index d46f0d4..572072e 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportRequest.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportRequest.java @@ -1,5 +1,7 @@ package io.cdap.plugin.marketo.common.api.entities.leads; +import io.cdap.plugin.marketo.common.api.entities.DateRange; + import java.util.List; import java.util.Map; @@ -7,19 +9,6 @@ * Represents leads bulk export request. */ public class LeadsExportRequest { - /** - * Represents date range. - */ - public static class DateRange { - String endAt = null; - String startAt = null; - - public DateRange(String startAt, String endAt) { - this.endAt = endAt; - this.startAt = startAt; - } - } - /** * Represents request filter. */ diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormat.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormat.java index 4fd514e..096b842 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormat.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoInputFormat.java @@ -16,26 +16,38 @@ package io.cdap.plugin.marketo.source.batch; +import com.google.gson.Gson; +import io.cdap.plugin.marketo.common.api.Helpers; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.InputFormat; import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.RecordReader; import org.apache.hadoop.mapreduce.TaskAttemptContext; -import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; /** * InputFormat for mapreduce job, which provides a single split of data. */ public class MarketoInputFormat extends InputFormat { + private static final Gson GSON = new Gson(); + @Override public List getSplits(JobContext jobContext) { - return Collections.singletonList(new NoOpMarketoSplit()); + Configuration conf = jobContext.getConfiguration(); + MarketoReportingSourceConfig config = GSON.fromJson( + conf.get(MarketoInputFormatProvider.PROPERTY_CONFIG_JSON), MarketoReportingSourceConfig.class); + + return Helpers.getDateRanges(config.getStartDate(), config.getEndDate()).stream() + .map(dateRange -> new MarketoReportingSplit(dateRange.getStartAt(), dateRange.getEndAt())) + .collect(Collectors.toList()); } @Override public RecordReader createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) { - return new MarketoRecordReader(); + MarketoReportingSplit split = (MarketoReportingSplit) inputSplit; + return new MarketoRecordReader(split.getBeginDate(), split.getEndDate()); } } diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java index a3b0e70..48e49e2 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java @@ -21,6 +21,7 @@ import io.cdap.cdap.api.data.schema.Schema; import io.cdap.plugin.marketo.common.api.LeadsExportJob; import io.cdap.plugin.marketo.common.api.Marketo; +import io.cdap.plugin.marketo.common.api.entities.DateRange; import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExportRequest; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; @@ -48,6 +49,13 @@ public class MarketoRecordReader extends RecordReader current = null; private Iterator iterator = null; + private String beginDate; + private String endDate; + + public MarketoRecordReader(String beginDate, String endDate) { + this.beginDate = beginDate; + this.endDate = endDate; + } @Override public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException { @@ -59,12 +67,18 @@ public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptCont List fields = config.getSchema().getFields().stream() .map(Schema.Field::getName).collect(Collectors.toList()); + DateRange dateRange = new DateRange(beginDate, endDate); LeadsExportRequest.ExportLeadFilter filter = LeadsExportRequest.ExportLeadFilter.builder() - .createdAt(new LeadsExportRequest.DateRange(config.getStartDate(), config.getEndDate())).build(); + .createdAt(dateRange).build(); LeadsExportRequest request = new LeadsExportRequest(fields, filter); LeadsExportJob job = marketo.exportLeads(request); - job.enqueue(); + LOG.info("Bulk leads export job with id '{}' has date range {}", job.getJobId(), dateRange); + + // TODO handle possible concurrent issues here, another mapper can take our slot + // wait for 10 minutes for available slot and enqueue job + marketo.onBulkExtractQueueAvailable(job::enqueue, 60 * 10); + try { job.waitCompletion(); } catch (InterruptedException ex) { @@ -72,10 +86,9 @@ public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptCont } String data = job.getFile(); - LOG.info(data); + // TODO stream here CSVParser parser = CSVFormat.DEFAULT.withHeader().parse(new StringReader(data)); iterator = parser.iterator(); -// iterator = config.getMarketo().iteratePage(config.getEntityType().getGetEndpoint()); } @Override diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/NoOpMarketoSplit.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSplit.java similarity index 58% rename from src/main/java/io/cdap/plugin/marketo/source/batch/NoOpMarketoSplit.java rename to src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSplit.java index 396ad97..7e12e27 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/NoOpMarketoSplit.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSplit.java @@ -21,20 +21,33 @@ import java.io.DataInput; import java.io.DataOutput; +import java.io.IOException; /** * A no-op split. */ -public class NoOpMarketoSplit extends InputSplit implements Writable { - public NoOpMarketoSplit() { +public class MarketoReportingSplit extends InputSplit implements Writable { + private String beginDate; + private String endDate; + + public MarketoReportingSplit() { + } + + public MarketoReportingSplit(String beginDate, String endDate) { + this.beginDate = beginDate; + this.endDate = endDate; } @Override - public void readFields(DataInput dataInput) { + public void readFields(DataInput dataInput) throws IOException { + beginDate = dataInput.readUTF(); + endDate = dataInput.readUTF(); } @Override - public void write(DataOutput dataOutput) { + public void write(DataOutput dataOutput) throws IOException { + dataOutput.writeUTF(beginDate); + dataOutput.writeUTF(endDate); } @Override @@ -46,4 +59,12 @@ public long getLength() { public String[] getLocations() { return new String[0]; } + + public String getBeginDate() { + return beginDate; + } + + public String getEndDate() { + return endDate; + } } From c7a0f317e8fc003b0c70f56d773711541ad1dbed Mon Sep 17 00:00:00 2001 From: Yevhenii Chekanskyi Date: Wed, 27 Nov 2019 17:32:28 +0200 Subject: [PATCH 06/12] PLUGIN-75 Marketo Plugin - add activities export. --- pom.xml | 1 - .../plugin/marketo/common/api/Helpers.java | 2 +- .../marketo/common/api/LeadsExportJob.java | 88 ------------- .../plugin/marketo/common/api/Marketo.java | 66 ++++++++-- .../marketo/common/api/MarketoHttp.java | 6 +- .../cdap/plugin/marketo/common/api/Urls.java | 8 +- .../entities/activities/ActivitiesExport.java | 86 +++++++++++++ .../activities/ActivitiesExportRequest.java | 20 +++ .../activities/ActivityTypeResponse.java | 79 ++++++++++++ .../activities/ExportActivityFilter.java | 59 +++++++++ .../common/api/job/AbstractBulkExportJob.java | 116 ++++++++++++++++++ .../common/api/job/ActivitiesExportJob.java | 55 +++++++++ .../common/api/job/LeadsExportJob.java | 55 +++++++++ .../source/batch/MarketoRecordReader.java | 35 ++++-- .../source/batch/MarketoReportingPlugin.java | 11 +- .../batch/MarketoReportingSchemaHelper.java | 55 +++++++++ .../batch/MarketoReportingSourceConfig.java | 26 ++-- .../marketo/source/batch/ReportType.java | 24 ++++ 18 files changed, 651 insertions(+), 141 deletions(-) delete mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExport.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExportRequest.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivityTypeResponse.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ExportActivityFilter.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/job/AbstractBulkExportJob.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/job/ActivitiesExportJob.java create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/job/LeadsExportJob.java create mode 100644 src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSchemaHelper.java create mode 100644 src/main/java/io/cdap/plugin/marketo/source/batch/ReportType.java diff --git a/pom.xml b/pom.xml index b697a2b..e3e9248 100644 --- a/pom.xml +++ b/pom.xml @@ -264,7 +264,6 @@ - junit diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java b/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java index 7301043..7418bb8 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java @@ -83,7 +83,7 @@ public static List getDateRanges(String beginDate, String endDate) { while (currentStart.compareTo(end) < 0) { OffsetDateTime nextEnd = currentStart.plusDays(30); result.add(new DateRange(currentStart.toString(), - min(nextEnd.minusSeconds(1), end).toString())); + min(nextEnd.minusSeconds(1), end).toString())); currentStart = nextEnd; } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java b/src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java deleted file mode 100644 index 9d2c13b..0000000 --- a/src/main/java/io/cdap/plugin/marketo/common/api/LeadsExportJob.java +++ /dev/null @@ -1,88 +0,0 @@ -package io.cdap.plugin.marketo.common.api; - -import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExport; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.TimeUnit; - -/** - * Leads export job. - */ -public class LeadsExportJob { - private static final Logger LOG = LoggerFactory.getLogger(LeadsExportJob.class); - private static final List WAIT_ABLE_STATE = Arrays.asList("Queued", "Processing"); - private static final String ENQUEUE_ABLE_STATUS = "Created"; - private static final String COMPLETED_STATUS = "Completed"; - - private String jobId; - private LeadsExport.ExportResponse lastStatus; - private Marketo marketo; - - public LeadsExportJob(LeadsExport.ExportResponse lastStatus, Marketo marketo) { - this.jobId = lastStatus.getExportId(); - this.lastStatus = lastStatus; - this.marketo = marketo; - LOG.info("BULK LEADS EXPORT - created job '{}'", this.jobId); - } - - public String getLastStatus() { - return lastStatus.getStatus(); - } - - public void waitCompletion() throws InterruptedException { - if (!WAIT_ABLE_STATE.contains(getLastStatus())) { - throw new IllegalStateException("Job must be enqueued before waiting for completion."); - } - - do { - Thread.sleep(TimeUnit.SECONDS.toMillis(30)); - LeadsExport.ExportResponse newState = marketo.leadsExportJobStatus(jobId); - logStatusChange(getLastStatus(), newState.getStatus()); - lastStatus = newState; - } while (WAIT_ABLE_STATE.contains(getLastStatus())); - - if (!getLastStatus().equals(COMPLETED_STATUS)) { - throw new IllegalStateException("Job expected to be in Completed state, but was in " + getLastStatus()); - } - } - - public void enqueue() { - if (!getLastStatus().equals(ENQUEUE_ABLE_STATUS)) { - throw new IllegalStateException("Job must be in Created status before enqueuing, but was in " + getLastStatus()); - } - - LeadsExport.ExportResponse newState = marketo.validatedPost( - String.format(Urls.BULK_EXPORT_LEADS_ENQUEUE, jobId), - Collections.emptyMap(), - inputStream -> Helpers.streamToObject(inputStream, LeadsExport.class), - null, null).singleExport(); - - logStatusChange(getLastStatus(), newState.getStatus()); - - if (!(newState.getStatus().equals("Queued") || newState.getStatus().equals("Processing"))) { - throw new IllegalStateException( - String.format("Expected Queued|Processing state for job '%s' but got '%s'", jobId, newState.getStatus())); - } - - lastStatus = newState; - } - - private void logStatusChange(String oldStatus, String newStatus) { - if (!oldStatus.equals(newStatus)) { - LOG.info("BULK LEADS EXPORT - job '{}' changed state '{}' -> '{}'", jobId, oldStatus, newStatus); - } - } - - public String getFile() { - return marketo.get(marketo.buildUri(String.format(Urls.BULK_EXPORT_LEADS_FILE, jobId), Collections.emptyMap()), - Helpers::streamToString); - } - - public String getJobId() { - return jobId; - } -} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java b/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java index d2bb9f9..ead5070 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java @@ -16,14 +16,21 @@ package io.cdap.plugin.marketo.common.api; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.gson.Gson; +import io.cdap.plugin.marketo.common.api.entities.activities.ActivitiesExport; +import io.cdap.plugin.marketo.common.api.entities.activities.ActivitiesExportRequest; +import io.cdap.plugin.marketo.common.api.entities.activities.ActivityTypeResponse; import io.cdap.plugin.marketo.common.api.entities.leads.LeadsDescribe; import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExport; import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExportRequest; +import io.cdap.plugin.marketo.common.api.job.ActivitiesExportJob; +import io.cdap.plugin.marketo.common.api.job.LeadsExportJob; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.InputStream; import java.util.Collections; import java.util.List; import java.util.Spliterator; @@ -38,6 +45,10 @@ public class Marketo extends MarketoHttp { private static final Logger LOG = LoggerFactory.getLogger(Marketo.class); static final Gson GSON = new Gson(); + public static final List ACTIVITY_FIELDS = ImmutableList.of("marketoGUID", "leadId", "activityDate", + "activityTypeId", "campaignId", + "primaryAttributeValueId", + "primaryAttributeValue", "attributes"); public Marketo(String marketoEndpoint, String clientId, String clientSecret) { super(marketoEndpoint, clientId, clientSecret); @@ -49,9 +60,15 @@ public List describeLeads() { Spliterator.ORDERED), false).collect(Collectors.toList()); } + public List describeBuildInActivities() { + return StreamSupport.stream(Spliterators.spliteratorUnknownSize( + iteratePage(Urls.BUILD_IN_ACTIVITIES_TYPES, ActivityTypeResponse.class, ActivityTypeResponse::getResult), + Spliterator.ORDERED), false).collect(Collectors.toList()); + } + public LeadsExportJob exportLeads(LeadsExportRequest request) { LeadsExport export = validatedPost(Urls.BULK_EXPORT_LEADS_CREATE, Collections.emptyMap(), - inputStream -> Helpers.streamToObject(inputStream, LeadsExport.class), + Marketo::streamToLeadsExport, request, GSON::toJson); return new LeadsExportJob(export.singleExport(), this); @@ -60,15 +77,30 @@ public LeadsExportJob exportLeads(LeadsExportRequest request) { public LeadsExport.ExportResponse leadsExportJobStatus(String jobId) { LeadsExport currentResp = validatedGet( String.format(Urls.BULK_EXPORT_LEADS_STATUS, jobId), - Collections.emptyMap(), inputStream -> Helpers.streamToObject(inputStream, LeadsExport.class)); + Collections.emptyMap(), Marketo::streamToLeadsExport); + return currentResp.singleExport(); + } + + public ActivitiesExportJob exportActivities(ActivitiesExportRequest request) { + ActivitiesExport export = validatedPost(Urls.BULK_EXPORT_ACTIVITIES_CREATE, Collections.emptyMap(), + Marketo::streamToActivitiesExport, + request, + GSON::toJson); + return new ActivitiesExportJob(export.singleExport(), this); + } + + public ActivitiesExport.ExportResponse activitiesExportJobStatus(String jobId) { + ActivitiesExport currentResp = validatedGet( + String.format(Urls.BULK_EXPORT_ACTIVITIES_STATUS, jobId), + Collections.emptyMap(), Marketo::streamToActivitiesExport); return currentResp.singleExport(); } /** * Waits until bulk extract queue has available slot and executes given action. * - * @param action action to execute once slot is available - * @param timeoutSeconds timeout, in seconds + * @param action action to execute once slot is available + * @param timeoutSeconds timeout in seconds */ public void onBulkExtractQueueAvailable(Runnable action, long timeoutSeconds) { long timeoutMillis = TimeUnit.SECONDS.toMillis(timeoutSeconds); @@ -88,14 +120,30 @@ public void onBulkExtractQueueAvailable(Runnable action, long timeoutSeconds) { throw new RuntimeException("Failed to get slot in bulk export queue - timeout"); } + public static LeadsExport streamToLeadsExport(InputStream inputStream) { + return Helpers.streamToObject(inputStream, LeadsExport.class); + } + + public static ActivitiesExport streamToActivitiesExport(InputStream inputStream) { + return Helpers.streamToObject(inputStream, ActivitiesExport.class); + } + private boolean canEnqueueJob() { - LeadsExport exportJobs = validatedGet(Urls.BULK_EXPORT_LEADS_LIST, - ImmutableMap.of("status", "queued,processing"), - inputStream -> Helpers.streamToObject(inputStream, LeadsExport.class) + LeadsExport leadsExportJobs = validatedGet(Urls.BULK_EXPORT_LEADS_LIST, + ImmutableMap.of("status", "queued,processing"), + Marketo::streamToLeadsExport ); - int jobsInQueue = exportJobs.getResult().size(); + + int jobsInQueue = leadsExportJobs.getResult().size(); + + ActivitiesExport activitiesExportJobs = validatedGet(Urls.BULK_EXPORT_ACTIVITIES_LIST, + ImmutableMap.of("status", "queued,processing"), + Marketo::streamToActivitiesExport + ); + jobsInQueue += activitiesExportJobs.getResult().size(); + LOG.debug("Jobs in queue: {}", jobsInQueue); - // TODO handle activity queue here + return jobsInQueue < 10; } } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java index f71cd77..bb94064 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java @@ -81,7 +81,7 @@ T validatedGet(String queryUrl, Map par }); } - T validatedPost(String queryUrl, Map parameters, + public T validatedPost(String queryUrl, Map parameters, Function deserializer, B body, Function qSerializer) { String logUri = "POST " + buildUri(queryUrl, parameters, false).toString(); @@ -128,7 +128,7 @@ private T retryableValidate(String logUri, Supplier return result; } - T get(URI uri, Function deserializer) { + public T get(URI uri, Function deserializer) { try (CloseableHttpClient httpClient = HttpClients.createDefault()) { HttpGet request = new HttpGet(uri); try (CloseableHttpResponse response = httpClient.execute(request, httpClientContext)) { @@ -157,7 +157,7 @@ private T post(URI uri, Function respDeserializer, B body } } - URI buildUri(String queryUrl, Map parameters) { + public URI buildUri(String queryUrl, Map parameters) { return buildUri(queryUrl, parameters, true); } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Urls.java b/src/main/java/io/cdap/plugin/marketo/common/api/Urls.java index 2db9085..32478d3 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/Urls.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Urls.java @@ -4,10 +4,16 @@ * Marketo API urls. */ public class Urls { + public static final String LEADS_DESCRIBE = "/rest/v1/leads/describe.json"; public static final String BULK_EXPORT_LEADS_LIST = "/bulk/v1/leads/export.json"; public static final String BULK_EXPORT_LEADS_CREATE = "/bulk/v1/leads/export/create.json"; public static final String BULK_EXPORT_LEADS_ENQUEUE = "/bulk/v1/leads/export/%s/enqueue.json"; public static final String BULK_EXPORT_LEADS_STATUS = "/bulk/v1/leads/export/%s/status.json"; public static final String BULK_EXPORT_LEADS_FILE = "/bulk/v1/leads/export/%s/file.json"; - public static final String LEADS_DESCRIBE = "/rest/v1/leads/describe.json"; + public static final String BULK_EXPORT_ACTIVITIES_LIST = "/bulk/v1/activities/export.json"; + public static final String BULK_EXPORT_ACTIVITIES_CREATE = "/bulk/v1/activities/export/create.json"; + public static final String BULK_EXPORT_ACTIVITIES_ENQUEUE = "/bulk/v1/activities/export/%s/enqueue.json"; + public static final String BULK_EXPORT_ACTIVITIES_STATUS = "/bulk/v1/activities/export/%s/status.json"; + public static final String BULK_EXPORT_ACTIVITIES_FILE = "/bulk/v1/activities/export/%s/file.json"; + public static final String BUILD_IN_ACTIVITIES_TYPES = "/rest/v1/activities/types.json"; } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExport.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExport.java new file mode 100644 index 0000000..d8f92aa --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExport.java @@ -0,0 +1,86 @@ +package io.cdap.plugin.marketo.common.api.entities.activities; + +import io.cdap.plugin.marketo.common.api.entities.BaseResponse; + +import java.util.Collections; +import java.util.List; + +/** + * Represents activities bulk export response. + */ +public class ActivitiesExport extends BaseResponse { + /** + * Represents export response item. + */ + public static class ExportResponse { + String createdAt; + String errorMsg; + String exportId; + int fileSize; + String fileChecksum; + String finishedAt; + String format; + int numberOfRecords; + String queuedAt; + String startedAt; + String status; + + public String getCreatedAt() { + return createdAt; + } + + public String getErrorMsg() { + return errorMsg; + } + + public String getExportId() { + return exportId; + } + + public int getFileSize() { + return fileSize; + } + + public String getFileChecksum() { + return fileChecksum; + } + + public String getFinishedAt() { + return finishedAt; + } + + public String getFormat() { + return format; + } + + public int getNumberOfRecords() { + return numberOfRecords; + } + + public String getQueuedAt() { + return queuedAt; + } + + public String getStartedAt() { + return startedAt; + } + + public String getStatus() { + return status; + } + } + + List result = Collections.emptyList(); + + public ExportResponse singleExport() { + if (result.size() != 1) { + throw new IllegalStateException("Expected single export job result."); + } + return result.get(0); + } + + public List getResult() { + return result; + } + +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExportRequest.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExportRequest.java new file mode 100644 index 0000000..bb05924 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExportRequest.java @@ -0,0 +1,20 @@ +package io.cdap.plugin.marketo.common.api.entities.activities; + +import java.util.List; +import java.util.Map; + +/** + * Represents activities bulk export request. + */ +public class ActivitiesExportRequest { + + Map columnHeaderNames = null; + List fields = null; + ExportActivityFilter filter = null; + String format = "CSV"; + + public ActivitiesExportRequest(List fields, ExportActivityFilter filter) { + this.fields = fields; + this.filter = filter; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivityTypeResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivityTypeResponse.java new file mode 100644 index 0000000..dc73a2a --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivityTypeResponse.java @@ -0,0 +1,79 @@ +package io.cdap.plugin.marketo.common.api.entities.activities; + +import io.cdap.plugin.marketo.common.api.entities.BaseResponse; + +import java.util.Collections; +import java.util.List; + +/** + * Activity type response. + */ +public class ActivityTypeResponse extends BaseResponse { + /** + * Activity type attribute. + */ + public static class ActivityTypeAttribute { + private String apiName; + private String dataType; + private String name; + + public String getApiName() { + return apiName; + } + + public String getDataType() { + return dataType; + } + + public String getName() { + return name; + } + + @Override + public String toString() { + return String.format("%s(%s, %s)", getApiName(), getDataType(), getName()); + } + } + + /** + * Attribute type. + */ + public static class ActivityType { + private String apiName; + private List attributes; + private String description; + private Integer id; + private String name; + private ActivityTypeAttribute primaryAttribute; + + public String getApiName() { + return apiName; + } + + public List getAttributes() { + return attributes; + } + + public String getDescription() { + return description; + } + + public Integer getId() { + return id; + } + + public String getName() { + return name; + } + + public ActivityTypeAttribute getPrimaryAttribute() { + return primaryAttribute; + } + } + + List result = Collections.emptyList(); + + public List getResult() { + return result; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ExportActivityFilter.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ExportActivityFilter.java new file mode 100644 index 0000000..6f1b071 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ExportActivityFilter.java @@ -0,0 +1,59 @@ +package io.cdap.plugin.marketo.common.api.entities.activities; + +import io.cdap.plugin.marketo.common.api.entities.DateRange; + +import java.util.List; + +/** + * Represents request filter. + */ +public class ExportActivityFilter { + private List activityTypeIds; + private DateRange createdAt; + + /** + * Builder. + */ + public static class Builder { + + private List activityTypeIds = null; + private DateRange createdAt = null; + + public Builder() { + } + + Builder(List activityTypeIds, DateRange createdAt) { + this.activityTypeIds = activityTypeIds; + this.createdAt = createdAt; + } + + public Builder activityTypeIds(List activityTypeIds) { + this.activityTypeIds = activityTypeIds; + return Builder.this; + } + + public Builder addActivityTypeIds(Integer activityTypeIds) { + this.activityTypeIds.add(activityTypeIds); + return Builder.this; + } + + public Builder createdAt(DateRange createdAt) { + this.createdAt = createdAt; + return Builder.this; + } + + public ExportActivityFilter build() { + + return new ExportActivityFilter(this); + } + } + + private ExportActivityFilter(Builder builder) { + this.activityTypeIds = builder.activityTypeIds; + this.createdAt = builder.createdAt; + } + + public static Builder builder() { + return new Builder(); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/job/AbstractBulkExportJob.java b/src/main/java/io/cdap/plugin/marketo/common/api/job/AbstractBulkExportJob.java new file mode 100644 index 0000000..05e5a02 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/job/AbstractBulkExportJob.java @@ -0,0 +1,116 @@ +package io.cdap.plugin.marketo.common.api.job; + +import io.cdap.plugin.marketo.common.api.Helpers; +import io.cdap.plugin.marketo.common.api.Marketo; +import org.slf4j.Logger; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * Base bulk export job wrapper. + * + * @param object that represents job status + */ +public abstract class AbstractBulkExportJob { + private static final List WAIT_ABLE_STATE = Arrays.asList("Queued", "Processing"); + private static final String ENQUEUE_ABLE_STATUS = "Created"; + private static final String COMPLETED_STATUS = "Completed"; + + private String jobId; + private T lastState; + private Marketo marketo; + + /** + * @param jobId job id + * @param lastState last status + * @param marketo marketo api instance + */ + public AbstractBulkExportJob(String jobId, + T lastState, + Marketo marketo) { + + this.jobId = jobId; + this.lastState = lastState; + this.marketo = marketo; + getLogger().info("{} - created job '{}'", getLogPrefix(), this.jobId); + } + + public abstract Logger getLogger(); + + public abstract T getFreshState(); + + public abstract String getStateStatus(T state); + + public abstract String getFileUrlTemplate(); + + public abstract String getLogPrefix(); + + protected abstract T enqueueImpl(); + + public T getLastState() { + return lastState; + } + + public Marketo getMarketo() { + return marketo; + } + + public void waitCompletion() { + if (!WAIT_ABLE_STATE.contains(getStateStatus(getLastState()))) { + throw new IllegalStateException("Job must be enqueued before waiting for completion."); + } + + do { + try { + Thread.sleep(TimeUnit.SECONDS.toMillis(30)); + } catch (InterruptedException e) { + throw new IllegalStateException("Failed to wait for job completion - interrupted"); + } + + T newState = getFreshState(); + logStatusChange(getStateStatus(getLastState()), getStateStatus(newState)); + lastState = newState; + } while (WAIT_ABLE_STATE.contains(getStateStatus(getLastState()))); + + if (!getStateStatus(getLastState()).equals(COMPLETED_STATUS)) { + throw new IllegalStateException("Job expected to be in Completed state, but was in " + + getStateStatus(getLastState())); + } + } + + public void enqueue() { + if (!getStateStatus(getLastState()).equals(ENQUEUE_ABLE_STATUS)) { + throw new IllegalStateException("Job must be in Created status before enqueuing, but was in " + + getStateStatus(getLastState())); + } + + T newState = enqueueImpl(); + + logStatusChange(getStateStatus(getLastState()), getStateStatus(newState)); + + if (!(getStateStatus(newState).equals("Queued") || getStateStatus(newState).equals("Processing"))) { + throw new IllegalStateException( + String.format("Expected Queued|Processing state for job '%s' but got '%s'", jobId, getStateStatus(newState))); + } + + lastState = newState; + } + + private void logStatusChange(String oldStatus, String newStatus) { + if (!oldStatus.equals(newStatus)) { + getLogger().info("{} - job '{}' changed state '{}' -> '{}'", getLogPrefix(), jobId, oldStatus, newStatus); + } + } + + public String getFile() { + return marketo.get(marketo.buildUri(String.format(getFileUrlTemplate(), jobId), Collections.emptyMap()), + Helpers::streamToString); + } + + public String getJobId() { + return jobId; + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/job/ActivitiesExportJob.java b/src/main/java/io/cdap/plugin/marketo/common/api/job/ActivitiesExportJob.java new file mode 100644 index 0000000..de26917 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/job/ActivitiesExportJob.java @@ -0,0 +1,55 @@ +package io.cdap.plugin.marketo.common.api.job; + +import io.cdap.plugin.marketo.common.api.Helpers; +import io.cdap.plugin.marketo.common.api.Marketo; +import io.cdap.plugin.marketo.common.api.Urls; +import io.cdap.plugin.marketo.common.api.entities.activities.ActivitiesExport; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; + +/** + * Activities export job. + */ +public class ActivitiesExportJob extends AbstractBulkExportJob { + private static final Logger LOG = LoggerFactory.getLogger(ActivitiesExportJob.class); + + public ActivitiesExportJob(ActivitiesExport.ExportResponse lastState, Marketo marketo) { + super(lastState.getExportId(), lastState, marketo); + } + + @Override + public Logger getLogger() { + return LOG; + } + + @Override + public ActivitiesExport.ExportResponse getFreshState() { + return getMarketo().activitiesExportJobStatus(getJobId()); + } + + @Override + public String getStateStatus(ActivitiesExport.ExportResponse state) { + return state.getStatus(); + } + + @Override + public String getFileUrlTemplate() { + return Urls.BULK_EXPORT_ACTIVITIES_FILE; + } + + @Override + public String getLogPrefix() { + return "BULK ACTIVITIES EXPORT"; + } + + @Override + protected ActivitiesExport.ExportResponse enqueueImpl() { + return getMarketo().validatedPost( + String.format(Urls.BULK_EXPORT_ACTIVITIES_ENQUEUE, getJobId()), + Collections.emptyMap(), + inputStream -> Helpers.streamToObject(inputStream, ActivitiesExport.class), + null, null).singleExport(); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/job/LeadsExportJob.java b/src/main/java/io/cdap/plugin/marketo/common/api/job/LeadsExportJob.java new file mode 100644 index 0000000..e0bf289 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/job/LeadsExportJob.java @@ -0,0 +1,55 @@ +package io.cdap.plugin.marketo.common.api.job; + +import io.cdap.plugin.marketo.common.api.Helpers; +import io.cdap.plugin.marketo.common.api.Marketo; +import io.cdap.plugin.marketo.common.api.Urls; +import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExport; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; + +/** + * Leads export job. + */ +public class LeadsExportJob extends AbstractBulkExportJob { + private static final Logger LOG = LoggerFactory.getLogger(LeadsExportJob.class); + + public LeadsExportJob(LeadsExport.ExportResponse lastState, Marketo marketo) { + super(lastState.getExportId(), lastState, marketo); + } + + @Override + public Logger getLogger() { + return LOG; + } + + @Override + public LeadsExport.ExportResponse getFreshState() { + return getMarketo().leadsExportJobStatus(getJobId()); + } + + @Override + public String getStateStatus(LeadsExport.ExportResponse state) { + return state.getStatus(); + } + + @Override + public String getFileUrlTemplate() { + return Urls.BULK_EXPORT_LEADS_FILE; + } + + @Override + public String getLogPrefix() { + return "BULK LEADS EXPORT"; + } + + @Override + protected LeadsExport.ExportResponse enqueueImpl() { + return getMarketo().validatedPost( + String.format(Urls.BULK_EXPORT_LEADS_ENQUEUE, getJobId()), + Collections.emptyMap(), + inputStream -> Helpers.streamToObject(inputStream, LeadsExport.class), + null, null).singleExport(); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java index 48e49e2..30f9b0e 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java @@ -19,10 +19,12 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import io.cdap.cdap.api.data.schema.Schema; -import io.cdap.plugin.marketo.common.api.LeadsExportJob; import io.cdap.plugin.marketo.common.api.Marketo; import io.cdap.plugin.marketo.common.api.entities.DateRange; +import io.cdap.plugin.marketo.common.api.entities.activities.ActivitiesExportRequest; +import io.cdap.plugin.marketo.common.api.entities.activities.ExportActivityFilter; import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExportRequest; +import io.cdap.plugin.marketo.common.api.job.AbstractBulkExportJob; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVRecord; @@ -65,25 +67,32 @@ public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptCont Marketo marketo = config.getMarketo(); - List fields = config.getSchema().getFields().stream() - .map(Schema.Field::getName).collect(Collectors.toList()); DateRange dateRange = new DateRange(beginDate, endDate); - LeadsExportRequest.ExportLeadFilter filter = LeadsExportRequest.ExportLeadFilter.builder() - .createdAt(dateRange).build(); - LeadsExportRequest request = new LeadsExportRequest(fields, filter); + AbstractBulkExportJob job; + switch (config.getReportType()) { + case LEADS: + List leadsFields = config.getSchema().getFields().stream() + .map(Schema.Field::getName).collect(Collectors.toList()); + LeadsExportRequest.ExportLeadFilter leadsFilter = LeadsExportRequest.ExportLeadFilter.builder() + .createdAt(dateRange).build(); + job = marketo.exportLeads(new LeadsExportRequest(leadsFields, leadsFilter)); + break; + case ACTIVITIES: + ExportActivityFilter activitiesFilter = ExportActivityFilter.builder() + .createdAt(dateRange).build(); + job = marketo.exportActivities(new ActivitiesExportRequest(Marketo.ACTIVITY_FIELDS, activitiesFilter)); + break; + default: + throw new IllegalArgumentException("Invalid report type " + config.getReportType()); + } - LeadsExportJob job = marketo.exportLeads(request); - LOG.info("Bulk leads export job with id '{}' has date range {}", job.getJobId(), dateRange); + LOG.info("BULK EXPORT JOB - job '{}' has date range '{}'", job.getJobId(), dateRange); // TODO handle possible concurrent issues here, another mapper can take our slot // wait for 10 minutes for available slot and enqueue job marketo.onBulkExtractQueueAvailable(job::enqueue, 60 * 10); - try { - job.waitCompletion(); - } catch (InterruptedException ex) { - throw new RuntimeException(ex); - } + job.waitCompletion(); String data = job.getFile(); // TODO stream here diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingPlugin.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingPlugin.java index e7949e7..875c15a 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingPlugin.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingPlugin.java @@ -72,16 +72,7 @@ public void prepareRun(BatchSourceContext batchSourceContext) { @Override public void transform(KeyValue> input, Emitter emitter) { - StructuredRecord.Builder builder = StructuredRecord.builder(config.getSchema()); - Map inputMap = input.getValue(); - config.getSchema().getFields().forEach( - field -> { - if (inputMap.containsKey(field.getName())) { - builder.set(field.getName(), String.valueOf(inputMap.remove(field.getName()))); - } - } - ); - emitter.emit(builder.build()); + emitter.emit(MarketoReportingSchemaHelper.getRecord(config.getSchema(), input.getValue())); } private void validateConfiguration(FailureCollector failureCollector) { diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSchemaHelper.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSchemaHelper.java new file mode 100644 index 0000000..1edef60 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSchemaHelper.java @@ -0,0 +1,55 @@ +package io.cdap.plugin.marketo.source.batch; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.marketo.common.api.Marketo; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * Various methods to deal with schema and record. + */ +public class MarketoReportingSchemaHelper { + public static Schema getActivitySchema() { + List fields = Marketo.ACTIVITY_FIELDS.stream() + .map(s -> Schema.Field.of(s, Schema.nullableOf(Schema.of(Schema.Type.STRING)))) + .collect(Collectors.toList()); + return Schema.recordOf("activityRecord", fields); + } + + public static Schema getSchema(MarketoReportingSourceConfig config) { + switch (config.getReportType()) { + case LEADS: + List fields = config.getMarketo().describeLeads().stream().map( + leadAttribute -> { + if (leadAttribute.getRest() != null) { + return Schema.Field.of(leadAttribute.getRest().getName(), + Schema.nullableOf(Schema.of(Schema.Type.STRING))); + } else { + return null; + } + } + ).filter(Objects::nonNull).collect(Collectors.toList()); + + return Schema.recordOf("LeadsRecord", fields); + case ACTIVITIES: + return getActivitySchema(); + } + throw new IllegalArgumentException("Failed to get schema for type " + config.getReportType()); + } + + public static StructuredRecord getRecord(Schema schema, Map fields) { + StructuredRecord.Builder builder = StructuredRecord.builder(schema); + schema.getFields().forEach( + field -> { + if (fields.containsKey(field.getName())) { + builder.set(field.getName(), fields.get(field.getName())); + } + } + ); + return builder.build(); + } +} diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java index c9cc414..8377cd0 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java @@ -24,10 +24,6 @@ import io.cdap.plugin.marketo.common.api.Marketo; import io.cdap.plugin.marketo.common.api.entities.MarketoToken; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; - /** * Provides all required configuration for reading Marketo entities. */ @@ -38,6 +34,7 @@ public class MarketoReportingSourceConfig extends ReferencePluginConfig { public static final String PROPERTY_REST_API_ENDPOINT = "restApiEndpoint"; public static final String PROPERTY_REST_API_IDENTITY = "restApiIdentity"; public static final String PROPERTY_DAILY_API_LIMIT = "dailyApiLimit"; + public static final String PROPERTY_REPORT_TYPE = "reportType"; public static final String PROPERTY_REPORT_FORMAT = "reportFormat"; public static final String PROPERTY_START_DATE = "startDate"; public static final String PROPERTY_END_DATE = "endDate"; @@ -72,6 +69,11 @@ public class MarketoReportingSourceConfig extends ReferencePluginConfig { @Macro protected String dailyApiLimit; + @Name(PROPERTY_REPORT_TYPE) + @Description("Report type format, leads or activities.") + @Macro + protected String reportType; + @Name(PROPERTY_REPORT_FORMAT) @Description("Report format.") @Macro @@ -98,17 +100,7 @@ public MarketoReportingSourceConfig(String referenceName) { public Schema getSchema() { if (schema == null) { - List fields = getMarketo().describeLeads().stream().map( - leadAttribute -> { - if (leadAttribute.getRest() != null) { - return Schema.Field.of(leadAttribute.getRest().getName(), Schema.nullableOf(Schema.of(Schema.Type.STRING))); - } else { - return null; - } - } - ).filter(Objects::nonNull).collect(Collectors.toList()); - - schema = Schema.recordOf("LeadsRecord", fields); + schema = MarketoReportingSchemaHelper.getSchema(this); } return schema; } @@ -152,4 +144,8 @@ public String getStartDate() { public String getEndDate() { return endDate; } + + public ReportType getReportType() { + return ReportType.fromString(reportType); + } } diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/ReportType.java b/src/main/java/io/cdap/plugin/marketo/source/batch/ReportType.java new file mode 100644 index 0000000..59c2291 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/ReportType.java @@ -0,0 +1,24 @@ +package io.cdap.plugin.marketo.source.batch; + +/** + * Represents report type. + */ +public enum ReportType { + LEADS("leads"), + ACTIVITIES("activities"); + + private String type; + + ReportType(String type) { + this.type = type; + } + + public static ReportType fromString(String reportType) { + for (ReportType rt : ReportType.values()) { + if (rt.type.equals(reportType)) { + return rt; + } + } + throw new IllegalArgumentException("unknown report type: " + reportType); + } +} From b7b257c4536d04c7ca0eb301925a88b0d8333384 Mon Sep 17 00:00:00 2001 From: Yevhenii Chekanskyi Date: Thu, 28 Nov 2019 12:24:57 +0200 Subject: [PATCH 07/12] PLUGIN-75 Marketo Plugin - add licence headers. --- .../plugin/marketo/common/api/Helpers.java | 16 ++++++++ .../marketo/common/api/MarketoHttp.java | 16 ++++++++ .../common/api/MarketoPageIterator.java | 16 ++++++++ .../cdap/plugin/marketo/common/api/Urls.java | 16 ++++++++ .../common/api/entities/BaseResponse.java | 16 ++++++++ .../common/api/entities/DateRange.java | 16 ++++++++ .../marketo/common/api/entities/Error.java | 16 ++++++++ .../marketo/common/api/entities/Warning.java | 16 ++++++++ .../entities/activities/ActivitiesExport.java | 16 ++++++++ .../activities/ActivitiesExportRequest.java | 16 ++++++++ .../activities/ActivityTypeResponse.java | 16 ++++++++ .../activities/ExportActivityFilter.java | 16 ++++++++ .../api/entities/leads/LeadsDescribe.java | 16 ++++++++ .../api/entities/leads/LeadsExport.java | 16 ++++++++ .../entities/leads/LeadsExportRequest.java | 16 ++++++++ .../common/api/job/AbstractBulkExportJob.java | 16 ++++++++ .../common/api/job/ActivitiesExportJob.java | 16 ++++++++ .../common/api/job/LeadsExportJob.java | 16 ++++++++ .../batch/MarketoReportingSchemaHelper.java | 16 ++++++++ .../batch/MarketoReportingSourceConfig.java | 38 ------------------- .../marketo/source/batch/ReportType.java | 16 ++++++++ .../marketo/common/api/MarketoHttpTest.java | 16 ++++++++ 22 files changed, 336 insertions(+), 38 deletions(-) diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java b/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java index 7418bb8..d01e0ee 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api; import com.google.common.base.Strings; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java index bb94064..2ac5e80 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api; import com.google.common.base.Strings; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/MarketoPageIterator.java b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoPageIterator.java index 0c32729..1de07bf 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/MarketoPageIterator.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoPageIterator.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api; import io.cdap.plugin.marketo.common.api.entities.BaseResponse; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Urls.java b/src/main/java/io/cdap/plugin/marketo/common/api/Urls.java index 32478d3..32479bb 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/Urls.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Urls.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api; /** diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/BaseResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/BaseResponse.java index dc3deb0..2f679ca 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/BaseResponse.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/BaseResponse.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api.entities; import java.util.Collections; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/DateRange.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/DateRange.java index 3764947..9e71503 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/DateRange.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/DateRange.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api.entities; /** diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/Error.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/Error.java index 2c9a9e2..594e9cb 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/Error.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/Error.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api.entities; /** diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/Warning.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/Warning.java index 5722e51..b1167fc 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/Warning.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/Warning.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api.entities; /** diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExport.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExport.java index d8f92aa..6137d52 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExport.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExport.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api.entities.activities; import io.cdap.plugin.marketo.common.api.entities.BaseResponse; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExportRequest.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExportRequest.java index bb05924..31fc681 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExportRequest.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExportRequest.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api.entities.activities; import java.util.List; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivityTypeResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivityTypeResponse.java index dc73a2a..b853484 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivityTypeResponse.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivityTypeResponse.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api.entities.activities; import io.cdap.plugin.marketo.common.api.entities.BaseResponse; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ExportActivityFilter.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ExportActivityFilter.java index 6f1b071..5e225cf 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ExportActivityFilter.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ExportActivityFilter.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api.entities.activities; import io.cdap.plugin.marketo.common.api.entities.DateRange; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsDescribe.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsDescribe.java index e1c832b..ba445fa 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsDescribe.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsDescribe.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api.entities.leads; import io.cdap.plugin.marketo.common.api.entities.BaseResponse; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExport.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExport.java index 65f6312..09d9cd4 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExport.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExport.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api.entities.leads; import io.cdap.plugin.marketo.common.api.entities.BaseResponse; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportRequest.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportRequest.java index 572072e..e2035ba 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportRequest.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportRequest.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api.entities.leads; import io.cdap.plugin.marketo.common.api.entities.DateRange; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/job/AbstractBulkExportJob.java b/src/main/java/io/cdap/plugin/marketo/common/api/job/AbstractBulkExportJob.java index 05e5a02..f58109a 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/job/AbstractBulkExportJob.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/job/AbstractBulkExportJob.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api.job; import io.cdap.plugin.marketo.common.api.Helpers; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/job/ActivitiesExportJob.java b/src/main/java/io/cdap/plugin/marketo/common/api/job/ActivitiesExportJob.java index de26917..db5cdf3 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/job/ActivitiesExportJob.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/job/ActivitiesExportJob.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api.job; import io.cdap.plugin.marketo.common.api.Helpers; diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/job/LeadsExportJob.java b/src/main/java/io/cdap/plugin/marketo/common/api/job/LeadsExportJob.java index e0bf289..740a9d7 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/job/LeadsExportJob.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/job/LeadsExportJob.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api.job; import io.cdap.plugin.marketo.common.api.Helpers; diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSchemaHelper.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSchemaHelper.java index 1edef60..0a578fd 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSchemaHelper.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSchemaHelper.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.source.batch; import io.cdap.cdap.api.data.format.StructuredRecord; diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java index 8377cd0..e1460bd 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java @@ -22,28 +22,18 @@ import io.cdap.cdap.api.data.schema.Schema; import io.cdap.plugin.common.ReferencePluginConfig; import io.cdap.plugin.marketo.common.api.Marketo; -import io.cdap.plugin.marketo.common.api.entities.MarketoToken; /** * Provides all required configuration for reading Marketo entities. */ public class MarketoReportingSourceConfig extends ReferencePluginConfig { - public static final String PROPERTY_ENTITY_NAME = "entityName"; public static final String PROPERTY_CLIENT_ID = "clientId"; public static final String PROPERTY_CLIENT_SECRET = "clientSecret"; public static final String PROPERTY_REST_API_ENDPOINT = "restApiEndpoint"; - public static final String PROPERTY_REST_API_IDENTITY = "restApiIdentity"; - public static final String PROPERTY_DAILY_API_LIMIT = "dailyApiLimit"; public static final String PROPERTY_REPORT_TYPE = "reportType"; - public static final String PROPERTY_REPORT_FORMAT = "reportFormat"; public static final String PROPERTY_START_DATE = "startDate"; public static final String PROPERTY_END_DATE = "endDate"; - @Name(PROPERTY_ENTITY_NAME) - @Description("Marketo entity name to fetch.") - @Macro - protected String entityName; - @Name(PROPERTY_CLIENT_ID) @Description("Marketo Client ID.") @Macro @@ -59,26 +49,11 @@ public class MarketoReportingSourceConfig extends ReferencePluginConfig { @Macro protected String restApiEndpoint; - @Name(PROPERTY_REST_API_IDENTITY) - @Description("REST API identity.") - @Macro - protected String restApiIdentity; - - @Name(PROPERTY_DAILY_API_LIMIT) - @Description("Marketo enforced daily API limit.") - @Macro - protected String dailyApiLimit; - @Name(PROPERTY_REPORT_TYPE) @Description("Report type format, leads or activities.") @Macro protected String reportType; - @Name(PROPERTY_REPORT_FORMAT) - @Description("Report format.") - @Macro - protected String reportFormat; - @Name(PROPERTY_START_DATE) @Description("Start date for the report.") @Macro @@ -90,7 +65,6 @@ public class MarketoReportingSourceConfig extends ReferencePluginConfig { protected String endDate; - private transient MarketoToken token = null; private transient Schema schema = null; private transient Marketo marketo = null; @@ -125,18 +99,6 @@ public String getRestApiEndpoint() { return restApiEndpoint; } - public String getRestApiIdentity() { - return restApiIdentity; - } - - public String getDailyApiLimit() { - return dailyApiLimit; - } - - public String getReportFormat() { - return reportFormat; - } - public String getStartDate() { return startDate; } diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/ReportType.java b/src/main/java/io/cdap/plugin/marketo/source/batch/ReportType.java index 59c2291..1f19f6a 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/ReportType.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/ReportType.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.source.batch; /** diff --git a/src/test/java/io/cdap/plugin/marketo/common/api/MarketoHttpTest.java b/src/test/java/io/cdap/plugin/marketo/common/api/MarketoHttpTest.java index 3ad8844..80c7071 100644 --- a/src/test/java/io/cdap/plugin/marketo/common/api/MarketoHttpTest.java +++ b/src/test/java/io/cdap/plugin/marketo/common/api/MarketoHttpTest.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.marketo.common.api; import com.github.tomakehurst.wiremock.client.WireMock; From e90db264ed36e6bdfc59b5b8e8029794bf9fc2c5 Mon Sep 17 00:00:00 2001 From: Yevhenii Chekanskyi Date: Thu, 28 Nov 2019 12:49:59 +0200 Subject: [PATCH 08/12] PLUGIN-75 Marketo Plugin - properties and docs.. --- docs/MarketoReportingPlugin-batchsource.md | 26 ++++++++++ icons/MarketoReportingPlugin-batchsource.png | Bin 0 -> 996 bytes .../MarketoReportingPlugin-batchsource.json | 49 ++++++++++++++++-- 3 files changed, 72 insertions(+), 3 deletions(-) create mode 100644 docs/MarketoReportingPlugin-batchsource.md create mode 100644 icons/MarketoReportingPlugin-batchsource.png diff --git a/docs/MarketoReportingPlugin-batchsource.md b/docs/MarketoReportingPlugin-batchsource.md new file mode 100644 index 0000000..68bca09 --- /dev/null +++ b/docs/MarketoReportingPlugin-batchsource.md @@ -0,0 +1,26 @@ +# Marketo Reporting batch source + +Description +----------- +This plugin used to query Leads or Activities entities for specified date range from Marketo. + +Properties +---------- +### General + +**Reference Name:** Name used to uniquely identify this source for lineage, annotating metadata, etc. + +**Rest API endpoint:** Marketo rest API endpoint, unique for each client. +### Authentication + +**Client ID:** Client ID. + +**Client Secret:** Client secret. + +### Report + +**Report Type:** Type of report, leads or activities. + +**Start Date:** Start date of report, in ISO 8601 format(1997-07-16T19:20:30+01:00) + +**End Date:** End date of report, in ISO 8601 format(1997-07-16T19:20:30+01:00) \ No newline at end of file diff --git a/icons/MarketoReportingPlugin-batchsource.png b/icons/MarketoReportingPlugin-batchsource.png new file mode 100644 index 0000000000000000000000000000000000000000..4abe7e74fa2422e7bba39b77d633ece4ccce275f GIT binary patch literal 996 zcmVPx#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D19?eAK~!i%?U`Rl zQ&Akp&$-)_Ec-(dC7B6E_0)srgAronLv9*^270gIw$hNEq8BNuy-4&Jh!wh_?qXya z=w=B~4l*dRhh8!a0yC_vh{`|vcYpn+C*fwB(_Qr}_QCDm^F5bwKKK02xxytpAQjp3 zq|&ynthpovc3fR36~PyLE1Db^rNXwgEL{|Qh#-`bR7!}ABn5^=lfAYk`-XvaMFlq^ zb5OdS5_)}do8gscY$;{wf@mTl6}M%Um*3vNn%Y3=UP^^7EVqHmV#h(2hDDnhQ35qd zQxoHo$?*uw;%tS{zMJ*L43=A-0OSe&c2`wV>pm9G1Fwr6(bUpFiBQj)A%Fn90YY5> zwCBUbyQdzHEx`KbAsHQwIFdTn3=0tA$8QJm`!36Ur^i@4!*Z<=Eb~xlkaL0np7hj= z&p24Q^60`P5(&KP2K?a~YDgNwQchy=3@fzQ=_VXSrJy6_EQk@hKKc8!YFAT~1vf>NmeNjEy()~l?WJJcdN@&gCoglA>m0Hnz- zOODfRsc&~%AGUhT)3X=xiAqGiS2c!%uXDQ0ckeu~jJ$65Kn*xa~6^au@bx1lE6}G%=)$K$gQ4JpI8(T?nF>v8b8&zQJk@P z3c|jd{~NAEP+&w*ltU=d%A%;@NCf(WLNK@e{>c|Ch(9D_%l`cQ$1%M6Cge97Rx1D) SQdJ880000 Date: Thu, 28 Nov 2019 13:08:33 +0200 Subject: [PATCH 09/12] PLUGIN-75 Marketo Plugin - config validation. --- .../source/batch/MarketoReportingPlugin.java | 4 +- .../batch/MarketoReportingSourceConfig.java | 74 +++++++++++++++++++ .../MarketoReportingPlugin-batchsource.json | 2 +- 3 files changed, 78 insertions(+), 2 deletions(-) diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingPlugin.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingPlugin.java index 875c15a..82b71b8 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingPlugin.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingPlugin.java @@ -28,6 +28,7 @@ import io.cdap.cdap.etl.api.PipelineConfigurer; import io.cdap.cdap.etl.api.batch.BatchSource; import io.cdap.cdap.etl.api.batch.BatchSourceContext; +import io.cdap.plugin.common.IdUtils; import io.cdap.plugin.common.LineageRecorder; import org.apache.hadoop.io.NullWritable; @@ -48,7 +49,6 @@ public class MarketoReportingPlugin extends BatchSource> input, Emitter } private void validateConfiguration(FailureCollector failureCollector) { + IdUtils.validateReferenceName(config.referenceName, failureCollector); + config.validate(failureCollector); failureCollector.getOrThrowException(); } } diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java index e1460bd..50b98c3 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java @@ -16,13 +16,20 @@ package io.cdap.plugin.marketo.source.batch; +import com.google.common.base.Strings; import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Macro; import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.plugin.common.ReferencePluginConfig; +import io.cdap.plugin.marketo.common.api.Helpers; import io.cdap.plugin.marketo.common.api.Marketo; +import java.net.MalformedURLException; +import java.net.URL; +import java.time.format.DateTimeParseException; + /** * Provides all required configuration for reading Marketo entities. */ @@ -110,4 +117,71 @@ public String getEndDate() { public ReportType getReportType() { return ReportType.fromString(reportType); } + + void validate(FailureCollector failureCollector) { + validateDate(failureCollector); + validateReportType(failureCollector); + validateMarketoEndpoint(failureCollector); + validateSecrets(failureCollector); + } + + void validateDate(FailureCollector failureCollector) { + if (!(containsMacro(PROPERTY_START_DATE) && containsMacro(PROPERTY_END_DATE))) { + try { + Helpers.getDateRanges(getStartDate(), getEndDate()); + } catch (IllegalArgumentException ex) { + String message = String.format("Failed to validate dates: %s.", ex.getMessage()); + String correctiveAction = null; + if (ex.getMessage().contains("start date more than end date")) { + message = "Start date more than end date."; + correctiveAction = "Swap dates."; + } + failureCollector.addFailure(message, correctiveAction) + .withConfigProperty(PROPERTY_START_DATE).withConfigProperty(PROPERTY_END_DATE); + } catch (DateTimeParseException ex) { + failureCollector.addFailure("Failed to parse one of dates.", + "Correct dates to ISO 8601 format.") + .withConfigProperty(PROPERTY_START_DATE).withConfigProperty(PROPERTY_END_DATE); + } + } + } + + void validateReportType(FailureCollector failureCollector) { + if (!containsMacro(PROPERTY_REPORT_TYPE)) { + try { + getReportType(); + } catch (IllegalArgumentException ex) { + failureCollector.addFailure(String.format("Incorrect reporting type '%s'.", reportType), + "Set reporting type to 'activities' or 'leads'.") + .withConfigProperty(PROPERTY_REPORT_TYPE); + } + } + } + + void validateSecrets(FailureCollector failureCollector) { + if (!containsMacro(PROPERTY_CLIENT_ID) && Strings.isNullOrEmpty(getClientId())) { + failureCollector.addFailure("Client ID is null or empty.", + "Set Client ID to non empty string.") + .withConfigProperty(PROPERTY_CLIENT_ID); + } + + if (!containsMacro(PROPERTY_CLIENT_SECRET) && Strings.isNullOrEmpty(getClientSecret())) { + failureCollector.addFailure("Client Secret is null or empty.", + "Set Client Secret to non empty string.") + .withConfigProperty(PROPERTY_CLIENT_SECRET); + } + } + + void validateMarketoEndpoint(FailureCollector failureCollector) { + if (!containsMacro(PROPERTY_REST_API_ENDPOINT)) { + try { + new URL(getRestApiEndpoint()); + } catch (MalformedURLException e) { + failureCollector + .addFailure(String.format("Malformed Marketo Rest API endpoint URL '%s'.", getRestApiEndpoint()), + "Change URL to valid.") + .withConfigProperty(PROPERTY_REST_API_ENDPOINT); + } + } + } } diff --git a/widgets/MarketoReportingPlugin-batchsource.json b/widgets/MarketoReportingPlugin-batchsource.json index 1910011..96bf773 100644 --- a/widgets/MarketoReportingPlugin-batchsource.json +++ b/widgets/MarketoReportingPlugin-batchsource.json @@ -38,7 +38,7 @@ "label": "Report", "properties": [ { - "widget-type": "textbox", + "widget-type": "select", "label": "Report Type", "name": "reportType", "widget-attributes": { From 578b693752157303823690dff5be2c49bc9ee1c6 Mon Sep 17 00:00:00 2001 From: Yevhenii Chekanskyi Date: Thu, 28 Nov 2019 17:03:41 +0200 Subject: [PATCH 10/12] PLUGIN-75 Marketo Plugin - address review comments. --- checkstyle.xml | 25 ++-- docs/MarketoReportingPlugin-batchsource.md | 10 +- pom.xml | 78 +++++++++++- .../plugin/marketo/common/api/Helpers.java | 13 +- .../plugin/marketo/common/api/Marketo.java | 58 ++++----- .../marketo/common/api/MarketoHttp.java | 21 ++-- .../entities/activities/ActivitiesExport.java | 112 +++++++----------- .../activities/ActivitiesExportResponse.java | 43 +++++++ ...scribe.java => LeadsDescribeResponse.java} | 2 +- .../api/entities/leads/LeadsExport.java | 112 +++++++----------- .../entities/leads/LeadsExportResponse.java | 42 +++++++ .../common/api/job/ActivitiesExportJob.java | 13 +- .../common/api/job/LeadsExportJob.java | 13 +- .../source/batch/MarketoRecordReader.java | 13 +- .../source/batch/MarketoReportingPlugin.java | 3 +- .../batch/MarketoReportingSourceConfig.java | 54 +++++---- .../marketo/source/batch/ReportType.java | 2 +- .../MarketoReportingPlugin-batchsource.json | 10 +- 18 files changed, 378 insertions(+), 246 deletions(-) create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExportResponse.java rename src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/{LeadsDescribe.java => LeadsDescribeResponse.java} (96%) create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportResponse.java diff --git a/checkstyle.xml b/checkstyle.xml index fb35f2d..04c6eda 100644 --- a/checkstyle.xml +++ b/checkstyle.xml @@ -16,8 +16,8 @@ --> + "-//Puppy Crawl//DTD Check Configuration 1.3//EN" + "http://www.puppycrawl.com/dtds/configuration_1_3.dtd"> - - - + @@ -114,8 +112,8 @@ page at http://checkstyle.sourceforge.net/config.html --> - - + + --> + value="${com.puppycrawl.tools.checkstyle.checks.sizes.LineLength.ignorePattern}" + default="^(package .*;\s*)|(import .*;\s*)|( *\* *https?://.*)$"/> @@ -298,7 +296,7 @@ page at http://checkstyle.sourceforge.net/config.html --> some other variants which we don't publicized to promote consistency). --> + value="fall through|Fall through|fallthru|Fallthru|falls through|Falls through|fallthrough|Fallthrough|No break|NO break|no break|continue on"/> @@ -396,11 +394,4 @@ page at http://checkstyle.sourceforge.net/config.html --> - - - - - - - diff --git a/docs/MarketoReportingPlugin-batchsource.md b/docs/MarketoReportingPlugin-batchsource.md index 68bca09..5451214 100644 --- a/docs/MarketoReportingPlugin-batchsource.md +++ b/docs/MarketoReportingPlugin-batchsource.md @@ -1,8 +1,8 @@ -# Marketo Reporting batch source +# Marketo Reporting Batch Source Description ----------- -This plugin used to query Leads or Activities entities for specified date range from Marketo. +This plugin is used to query Leads or Activities entities for specified date range from Marketo. Properties ---------- @@ -19,8 +19,8 @@ Properties ### Report -**Report Type:** Type of report, leads or activities. +**Report Type:** Type of report. One of 'leads' or 'activities'. -**Start Date:** Start date of report, in ISO 8601 format(1997-07-16T19:20:30+01:00) +**Start Date:** Start date of report. In ISO 8601 format(1997-07-16T19:20:30+01:00). -**End Date:** End date of report, in ISO 8601 format(1997-07-16T19:20:30+01:00) \ No newline at end of file +**End Date:** End date of report. In ISO 8601 format(1997-07-16T19:20:30+01:00). \ No newline at end of file diff --git a/pom.xml b/pom.xml index e3e9248..e8a095b 100644 --- a/pom.xml +++ b/pom.xml @@ -1,12 +1,27 @@ + 4.0.0 - io.cdap + io.cdap.plugin marketo-entity-plugin - 1.0-SNAPSHOT + 1.0.0-SNAPSHOT @@ -21,7 +36,7 @@ 6.1.0-SNAPSHOT - 2.3.0 + 2.8.0 4.5.9 2.3.0-SNAPSHOT 2.1.3 @@ -281,6 +296,61 @@ + + org.apache.rat + apache-rat-plugin + 0.13 + + + org.apache.maven.doxia + doxia-core + 1.6 + + + xerces + xercesImpl + + + + + + + rat-check + validate + + check + + + + LICENSE*.txt + + *.rst + *.md + **/*.cdap + **/*.yaml + **/*.md + logs/** + .git/** + .idea/** + **/grok/patterns/** + conf/** + data/** + plugins/** + **/*.patch + **/logrotate.d/** + **/limits.d/** + **/*.json + **/*.json.template + **/MANIFEST.MF + + **/org/apache/hadoop/** + + **/resources/** + + + + + org.apache.maven.plugins maven-checkstyle-plugin @@ -307,7 +377,7 @@ com.puppycrawl.tools checkstyle - 6.19 + 8.18 diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java b/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java index d01e0ee..bda2387 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Helpers.java @@ -41,7 +41,7 @@ public static String streamToString(InputStream inputStream) { try { return IOUtils.toString(inputStream, StandardCharsets.UTF_8); } catch (IOException e) { - throw new RuntimeException(e); + throw new RuntimeException(String.format("Failed to read stream completely due to '%s'", e.getMessage())); } } @@ -49,13 +49,13 @@ public static T streamToObject(InputStream inputStream, Class cls) { return Marketo.GSON.fromJson(new InputStreamReader(inputStream), cls); } - public static RuntimeException failForUri(String method, URI uri, Exception ex) { + public static RuntimeException failForMethodAndUri(String method, URI uri, Exception ex) { String message = ex.getMessage(); if (Strings.isNullOrEmpty(message)) { if (ex.getCause() != null) { message = ex.getCause().getMessage(); if (Strings.isNullOrEmpty(message)) { - message = "failed to make request"; + message = "Unknown failure"; } } } @@ -66,9 +66,10 @@ public static RuntimeException failForUri(String method, URI uri, Exception ex) uriBuilder.setParameters(queryParameters); try { String uriString = uriBuilder.build().toString(); - return new RuntimeException(String.format("Failed %s %s - %s", method, uriString, message)); + return new RuntimeException(String.format("Failed '%s' '%s' - '%s'", method, uriString, message)); } catch (URISyntaxException e) { - return new RuntimeException(message); + // this will never happen since we rebuilding already validated uri, just make compiler happy + return new RuntimeException(e); } } @@ -85,7 +86,7 @@ public static List getDateRanges(String beginDate, String endDate) { OffsetDateTime end = OffsetDateTime.parse(endDate); if (start.compareTo(end) > 0) { - throw new IllegalArgumentException("start date more than end date"); + throw new IllegalArgumentException("Start date cannot be greater than the end date."); } int compareResult = start.plusDays(30).compareTo(end); diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java b/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java index ead5070..809702c 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java @@ -21,10 +21,12 @@ import com.google.gson.Gson; import io.cdap.plugin.marketo.common.api.entities.activities.ActivitiesExport; import io.cdap.plugin.marketo.common.api.entities.activities.ActivitiesExportRequest; +import io.cdap.plugin.marketo.common.api.entities.activities.ActivitiesExportResponse; import io.cdap.plugin.marketo.common.api.entities.activities.ActivityTypeResponse; -import io.cdap.plugin.marketo.common.api.entities.leads.LeadsDescribe; +import io.cdap.plugin.marketo.common.api.entities.leads.LeadsDescribeResponse; import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExport; import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExportRequest; +import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExportResponse; import io.cdap.plugin.marketo.common.api.job.ActivitiesExportJob; import io.cdap.plugin.marketo.common.api.job.LeadsExportJob; import org.slf4j.Logger; @@ -54,9 +56,9 @@ public Marketo(String marketoEndpoint, String clientId, String clientSecret) { super(marketoEndpoint, clientId, clientSecret); } - public List describeLeads() { + public List describeLeads() { return StreamSupport.stream(Spliterators.spliteratorUnknownSize( - iteratePage(Urls.LEADS_DESCRIBE, LeadsDescribe.class, LeadsDescribe::getResult), + iteratePage(Urls.LEADS_DESCRIBE, LeadsDescribeResponse.class, LeadsDescribeResponse::getResult), Spliterator.ORDERED), false).collect(Collectors.toList()); } @@ -67,30 +69,30 @@ public List describeBuildInActivities() { } public LeadsExportJob exportLeads(LeadsExportRequest request) { - LeadsExport export = validatedPost(Urls.BULK_EXPORT_LEADS_CREATE, Collections.emptyMap(), - Marketo::streamToLeadsExport, - request, - GSON::toJson); + LeadsExportResponse export = validatedPost(Urls.BULK_EXPORT_LEADS_CREATE, Collections.emptyMap(), + Marketo::streamToLeadsExport, + request, + GSON::toJson); return new LeadsExportJob(export.singleExport(), this); } - public LeadsExport.ExportResponse leadsExportJobStatus(String jobId) { - LeadsExport currentResp = validatedGet( + public LeadsExport leadsExportJobStatus(String jobId) { + LeadsExportResponse currentResp = validatedGet( String.format(Urls.BULK_EXPORT_LEADS_STATUS, jobId), Collections.emptyMap(), Marketo::streamToLeadsExport); return currentResp.singleExport(); } public ActivitiesExportJob exportActivities(ActivitiesExportRequest request) { - ActivitiesExport export = validatedPost(Urls.BULK_EXPORT_ACTIVITIES_CREATE, Collections.emptyMap(), - Marketo::streamToActivitiesExport, - request, - GSON::toJson); + ActivitiesExportResponse export = validatedPost(Urls.BULK_EXPORT_ACTIVITIES_CREATE, Collections.emptyMap(), + Marketo::streamToActivitiesExport, + request, + GSON::toJson); return new ActivitiesExportJob(export.singleExport(), this); } - public ActivitiesExport.ExportResponse activitiesExportJobStatus(String jobId) { - ActivitiesExport currentResp = validatedGet( + public ActivitiesExport activitiesExportJobStatus(String jobId) { + ActivitiesExportResponse currentResp = validatedGet( String.format(Urls.BULK_EXPORT_ACTIVITIES_STATUS, jobId), Collections.emptyMap(), Marketo::streamToActivitiesExport); return currentResp.singleExport(); @@ -117,30 +119,30 @@ public void onBulkExtractQueueAvailable(Runnable action, long timeoutSeconds) { } } } - throw new RuntimeException("Failed to get slot in bulk export queue - timeout"); + throw new RuntimeException("Failed to get slot in bulk export queue due to timeout"); } - public static LeadsExport streamToLeadsExport(InputStream inputStream) { - return Helpers.streamToObject(inputStream, LeadsExport.class); + public static LeadsExportResponse streamToLeadsExport(InputStream inputStream) { + return Helpers.streamToObject(inputStream, LeadsExportResponse.class); } - public static ActivitiesExport streamToActivitiesExport(InputStream inputStream) { - return Helpers.streamToObject(inputStream, ActivitiesExport.class); + public static ActivitiesExportResponse streamToActivitiesExport(InputStream inputStream) { + return Helpers.streamToObject(inputStream, ActivitiesExportResponse.class); } private boolean canEnqueueJob() { - LeadsExport leadsExportJobs = validatedGet(Urls.BULK_EXPORT_LEADS_LIST, - ImmutableMap.of("status", "queued,processing"), - Marketo::streamToLeadsExport + LeadsExportResponse leadsExportResponseJobs = validatedGet(Urls.BULK_EXPORT_LEADS_LIST, + ImmutableMap.of("status", "queued,processing"), + Marketo::streamToLeadsExport ); - int jobsInQueue = leadsExportJobs.getResult().size(); + int jobsInQueue = leadsExportResponseJobs.getResult().size(); - ActivitiesExport activitiesExportJobs = validatedGet(Urls.BULK_EXPORT_ACTIVITIES_LIST, - ImmutableMap.of("status", "queued,processing"), - Marketo::streamToActivitiesExport + ActivitiesExportResponse activitiesExportResponceJobs = validatedGet(Urls.BULK_EXPORT_ACTIVITIES_LIST, + ImmutableMap.of("status", "queued,processing"), + Marketo::streamToActivitiesExport ); - jobsInQueue += activitiesExportJobs.getResult().size(); + jobsInQueue += activitiesExportResponceJobs.getResult().size(); LOG.debug("Jobs in queue: {}", jobsInQueue); diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java index 2ac5e80..c4e7cf1 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java @@ -98,8 +98,8 @@ T validatedGet(String queryUrl, Map par } public T validatedPost(String queryUrl, Map parameters, - Function deserializer, - B body, Function qSerializer) { + Function deserializer, + B body, Function qSerializer) { String logUri = "POST " + buildUri(queryUrl, parameters, false).toString(); return retryableValidate(logUri, () -> { URI queryUri = buildUri(queryUrl, parameters, true); @@ -148,13 +148,11 @@ public T get(URI uri, Function deserializer) { try (CloseableHttpClient httpClient = HttpClients.createDefault()) { HttpGet request = new HttpGet(uri); try (CloseableHttpResponse response = httpClient.execute(request, httpClientContext)) { - if (response.getStatusLine().getStatusCode() >= 300) { - throw new IOException(IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8)); - } + checkResponseCode(response); return deserializer.apply(response.getEntity().getContent()); } } catch (Exception e) { - throw Helpers.failForUri("GET", uri, e); + throw Helpers.failForMethodAndUri("GET", uri, e); } } @@ -166,10 +164,19 @@ private T post(URI uri, Function respDeserializer, B body request.setEntity(new StringEntity(qSerializer.apply(body), ContentType.APPLICATION_JSON)); } try (CloseableHttpResponse response = httpClient.execute(request, httpClientContext)) { + checkResponseCode(response); return respDeserializer.apply(response.getEntity().getContent()); } } catch (Exception e) { - throw Helpers.failForUri("POST", uri, e); + throw Helpers.failForMethodAndUri("POST", uri, e); + } + } + + private static void checkResponseCode(CloseableHttpResponse response) throws IOException { + int statusCode = response.getStatusLine().getStatusCode(); + if (statusCode >= 300) { + String responseBody = IOUtils.toString(response.getEntity().getContent(), StandardCharsets.UTF_8); + throw new RuntimeException(String.format("Http code '%s', response '%s'", statusCode, responseBody)); } } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExport.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExport.java index 6137d52..2cfc72e 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExport.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExport.java @@ -16,87 +16,63 @@ package io.cdap.plugin.marketo.common.api.entities.activities; -import io.cdap.plugin.marketo.common.api.entities.BaseResponse; - -import java.util.Collections; -import java.util.List; - /** - * Represents activities bulk export response. + * Represents export response item. */ -public class ActivitiesExport extends BaseResponse { - /** - * Represents export response item. - */ - public static class ExportResponse { - String createdAt; - String errorMsg; - String exportId; - int fileSize; - String fileChecksum; - String finishedAt; - String format; - int numberOfRecords; - String queuedAt; - String startedAt; - String status; - - public String getCreatedAt() { - return createdAt; - } - - public String getErrorMsg() { - return errorMsg; - } - - public String getExportId() { - return exportId; - } - - public int getFileSize() { - return fileSize; - } - - public String getFileChecksum() { - return fileChecksum; - } +public class ActivitiesExport { + String createdAt; + String errorMsg; + String exportId; + int fileSize; + String fileChecksum; + String finishedAt; + String format; + int numberOfRecords; + String queuedAt; + String startedAt; + String status; + + public String getCreatedAt() { + return createdAt; + } - public String getFinishedAt() { - return finishedAt; - } + public String getErrorMsg() { + return errorMsg; + } - public String getFormat() { - return format; - } + public String getExportId() { + return exportId; + } - public int getNumberOfRecords() { - return numberOfRecords; - } + public int getFileSize() { + return fileSize; + } - public String getQueuedAt() { - return queuedAt; - } + public String getFileChecksum() { + return fileChecksum; + } - public String getStartedAt() { - return startedAt; - } + public String getFinishedAt() { + return finishedAt; + } - public String getStatus() { - return status; - } + public String getFormat() { + return format; } - List result = Collections.emptyList(); + public int getNumberOfRecords() { + return numberOfRecords; + } - public ExportResponse singleExport() { - if (result.size() != 1) { - throw new IllegalStateException("Expected single export job result."); - } - return result.get(0); + public String getQueuedAt() { + return queuedAt; } - public List getResult() { - return result; + public String getStartedAt() { + return startedAt; } + public String getStatus() { + return status; + } } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExportResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExportResponse.java new file mode 100644 index 0000000..0b19058 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/activities/ActivitiesExportResponse.java @@ -0,0 +1,43 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.activities; + +import io.cdap.plugin.marketo.common.api.entities.BaseResponse; + +import java.util.Collections; +import java.util.List; + +/** + * Represents activities bulk export response. + */ +public class ActivitiesExportResponse extends BaseResponse { + + List result = Collections.emptyList(); + + public ActivitiesExport singleExport() { + if (result.size() != 1) { + throw new IllegalStateException( + String.format("Expected single export job result, but found '%s' results.", result.size())); + } + return result.get(0); + } + + public List getResult() { + return result; + } + +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsDescribe.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsDescribeResponse.java similarity index 96% rename from src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsDescribe.java rename to src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsDescribeResponse.java index ba445fa..afd89ec 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsDescribe.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsDescribeResponse.java @@ -24,7 +24,7 @@ /** * Represents leads describe response. */ -public class LeadsDescribe extends BaseResponse { +public class LeadsDescribeResponse extends BaseResponse { /** * Represents lead field description. */ diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExport.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExport.java index 09d9cd4..f833207 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExport.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExport.java @@ -16,87 +16,63 @@ package io.cdap.plugin.marketo.common.api.entities.leads; -import io.cdap.plugin.marketo.common.api.entities.BaseResponse; - -import java.util.Collections; -import java.util.List; - /** - * Represents leads bulk export response. + * Represents export response item. */ -public class LeadsExport extends BaseResponse { - /** - * Represents export response item. - */ - public static class ExportResponse { - String createdAt; - String errorMsg; - String exportId; - int fileSize; - String fileChecksum; - String finishedAt; - String format; - int numberOfRecords; - String queuedAt; - String startedAt; - String status; - - public String getCreatedAt() { - return createdAt; - } - - public String getErrorMsg() { - return errorMsg; - } - - public String getExportId() { - return exportId; - } - - public int getFileSize() { - return fileSize; - } - - public String getFileChecksum() { - return fileChecksum; - } +public class LeadsExport { + String createdAt; + String errorMsg; + String exportId; + int fileSize; + String fileChecksum; + String finishedAt; + String format; + int numberOfRecords; + String queuedAt; + String startedAt; + String status; + + public String getCreatedAt() { + return createdAt; + } - public String getFinishedAt() { - return finishedAt; - } + public String getErrorMsg() { + return errorMsg; + } - public String getFormat() { - return format; - } + public String getExportId() { + return exportId; + } - public int getNumberOfRecords() { - return numberOfRecords; - } + public int getFileSize() { + return fileSize; + } - public String getQueuedAt() { - return queuedAt; - } + public String getFileChecksum() { + return fileChecksum; + } - public String getStartedAt() { - return startedAt; - } + public String getFinishedAt() { + return finishedAt; + } - public String getStatus() { - return status; - } + public String getFormat() { + return format; } - List result = Collections.emptyList(); + public int getNumberOfRecords() { + return numberOfRecords; + } - public ExportResponse singleExport() { - if (result.size() != 1) { - throw new IllegalStateException("Expected single export job result."); - } - return result.get(0); + public String getQueuedAt() { + return queuedAt; } - public List getResult() { - return result; + public String getStartedAt() { + return startedAt; } + public String getStatus() { + return status; + } } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportResponse.java b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportResponse.java new file mode 100644 index 0000000..bb58467 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/entities/leads/LeadsExportResponse.java @@ -0,0 +1,42 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api.entities.leads; + +import io.cdap.plugin.marketo.common.api.entities.BaseResponse; + +import java.util.Collections; +import java.util.List; + +/** + * Represents leads bulk export response. + */ +public class LeadsExportResponse extends BaseResponse { + + List result = Collections.emptyList(); + + public LeadsExport singleExport() { + if (result.size() != 1) { + throw new IllegalStateException("Expected single export job result."); + } + return result.get(0); + } + + public List getResult() { + return result; + } + +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/job/ActivitiesExportJob.java b/src/main/java/io/cdap/plugin/marketo/common/api/job/ActivitiesExportJob.java index db5cdf3..7a6c35d 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/job/ActivitiesExportJob.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/job/ActivitiesExportJob.java @@ -20,6 +20,7 @@ import io.cdap.plugin.marketo.common.api.Marketo; import io.cdap.plugin.marketo.common.api.Urls; import io.cdap.plugin.marketo.common.api.entities.activities.ActivitiesExport; +import io.cdap.plugin.marketo.common.api.entities.activities.ActivitiesExportResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,10 +29,10 @@ /** * Activities export job. */ -public class ActivitiesExportJob extends AbstractBulkExportJob { +public class ActivitiesExportJob extends AbstractBulkExportJob { private static final Logger LOG = LoggerFactory.getLogger(ActivitiesExportJob.class); - public ActivitiesExportJob(ActivitiesExport.ExportResponse lastState, Marketo marketo) { + public ActivitiesExportJob(ActivitiesExport lastState, Marketo marketo) { super(lastState.getExportId(), lastState, marketo); } @@ -41,12 +42,12 @@ public Logger getLogger() { } @Override - public ActivitiesExport.ExportResponse getFreshState() { + public ActivitiesExport getFreshState() { return getMarketo().activitiesExportJobStatus(getJobId()); } @Override - public String getStateStatus(ActivitiesExport.ExportResponse state) { + public String getStateStatus(ActivitiesExport state) { return state.getStatus(); } @@ -61,11 +62,11 @@ public String getLogPrefix() { } @Override - protected ActivitiesExport.ExportResponse enqueueImpl() { + protected ActivitiesExport enqueueImpl() { return getMarketo().validatedPost( String.format(Urls.BULK_EXPORT_ACTIVITIES_ENQUEUE, getJobId()), Collections.emptyMap(), - inputStream -> Helpers.streamToObject(inputStream, ActivitiesExport.class), + inputStream -> Helpers.streamToObject(inputStream, ActivitiesExportResponse.class), null, null).singleExport(); } } diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/job/LeadsExportJob.java b/src/main/java/io/cdap/plugin/marketo/common/api/job/LeadsExportJob.java index 740a9d7..f96587f 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/job/LeadsExportJob.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/job/LeadsExportJob.java @@ -20,6 +20,7 @@ import io.cdap.plugin.marketo.common.api.Marketo; import io.cdap.plugin.marketo.common.api.Urls; import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExport; +import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExportResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,10 +29,10 @@ /** * Leads export job. */ -public class LeadsExportJob extends AbstractBulkExportJob { +public class LeadsExportJob extends AbstractBulkExportJob { private static final Logger LOG = LoggerFactory.getLogger(LeadsExportJob.class); - public LeadsExportJob(LeadsExport.ExportResponse lastState, Marketo marketo) { + public LeadsExportJob(LeadsExport lastState, Marketo marketo) { super(lastState.getExportId(), lastState, marketo); } @@ -41,12 +42,12 @@ public Logger getLogger() { } @Override - public LeadsExport.ExportResponse getFreshState() { + public LeadsExport getFreshState() { return getMarketo().leadsExportJobStatus(getJobId()); } @Override - public String getStateStatus(LeadsExport.ExportResponse state) { + public String getStateStatus(LeadsExport state) { return state.getStatus(); } @@ -61,11 +62,11 @@ public String getLogPrefix() { } @Override - protected LeadsExport.ExportResponse enqueueImpl() { + protected LeadsExport enqueueImpl() { return getMarketo().validatedPost( String.format(Urls.BULK_EXPORT_LEADS_ENQUEUE, getJobId()), Collections.emptyMap(), - inputStream -> Helpers.streamToObject(inputStream, LeadsExport.class), + inputStream -> Helpers.streamToObject(inputStream, LeadsExportResponse.class), null, null).singleExport(); } } diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java index 30f9b0e..f6b306b 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java @@ -72,14 +72,20 @@ public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptCont switch (config.getReportType()) { case LEADS: List leadsFields = config.getSchema().getFields().stream() - .map(Schema.Field::getName).collect(Collectors.toList()); + .map(Schema.Field::getName) + .collect(Collectors.toList()); + LeadsExportRequest.ExportLeadFilter leadsFilter = LeadsExportRequest.ExportLeadFilter.builder() - .createdAt(dateRange).build(); + .createdAt(dateRange) + .build(); + job = marketo.exportLeads(new LeadsExportRequest(leadsFields, leadsFilter)); break; case ACTIVITIES: ExportActivityFilter activitiesFilter = ExportActivityFilter.builder() - .createdAt(dateRange).build(); + .createdAt(dateRange) + .build(); + job = marketo.exportActivities(new ActivitiesExportRequest(Marketo.ACTIVITY_FIELDS, activitiesFilter)); break; default: @@ -104,7 +110,6 @@ public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptCont public boolean nextKeyValue() { if (iterator.hasNext()) { current = iterator.next().toMap(); - LOG.debug("Got record '{}'", current.toString()); return true; } return false; diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingPlugin.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingPlugin.java index 82b71b8..585814d 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingPlugin.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingPlugin.java @@ -41,7 +41,7 @@ */ @Plugin(type = BatchSource.PLUGIN_TYPE) @Name(MarketoReportingPlugin.NAME) -@Description("Reads entities from Marketo.") +@Description("Reads Leads or Activities from Marketo.") public class MarketoReportingPlugin extends BatchSource, StructuredRecord> { public static final String NAME = "MarketoReportingPlugin"; @@ -76,7 +76,6 @@ public void transform(KeyValue> input, Emitter } private void validateConfiguration(FailureCollector failureCollector) { - IdUtils.validateReferenceName(config.referenceName, failureCollector); config.validate(failureCollector); failureCollector.getOrThrowException(); } diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java index 50b98c3..e7282aa 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoReportingSourceConfig.java @@ -22,12 +22,13 @@ import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.plugin.common.IdUtils; import io.cdap.plugin.common.ReferencePluginConfig; -import io.cdap.plugin.marketo.common.api.Helpers; import io.cdap.plugin.marketo.common.api.Marketo; import java.net.MalformedURLException; import java.net.URL; +import java.time.OffsetDateTime; import java.time.format.DateTimeParseException; /** @@ -71,7 +72,6 @@ public class MarketoReportingSourceConfig extends ReferencePluginConfig { @Macro protected String endDate; - private transient Schema schema = null; private transient Marketo marketo = null; @@ -93,7 +93,6 @@ public Marketo getMarketo() { return marketo; } - public String getClientId() { return clientId; } @@ -119,6 +118,7 @@ public ReportType getReportType() { } void validate(FailureCollector failureCollector) { + IdUtils.validateReferenceName(referenceName, failureCollector); validateDate(failureCollector); validateReportType(failureCollector); validateMarketoEndpoint(failureCollector); @@ -126,22 +126,37 @@ void validate(FailureCollector failureCollector) { } void validateDate(FailureCollector failureCollector) { + if (!containsMacro(PROPERTY_START_DATE)) { + try { + OffsetDateTime.parse(getStartDate()); + } catch (DateTimeParseException ex) { + failureCollector.addFailure("Failed to parse start date.", + "Correct date to ISO 8601 format.") + .withConfigProperty(PROPERTY_START_DATE); + } + } + + if (!containsMacro(PROPERTY_END_DATE)) { + try { + OffsetDateTime.parse(getStartDate()); + } catch (DateTimeParseException ex) { + failureCollector.addFailure("Failed to parse end date.", + "Correct date to ISO 8601 format.") + .withConfigProperty(PROPERTY_END_DATE); + } + } + if (!(containsMacro(PROPERTY_START_DATE) && containsMacro(PROPERTY_END_DATE))) { try { - Helpers.getDateRanges(getStartDate(), getEndDate()); - } catch (IllegalArgumentException ex) { - String message = String.format("Failed to validate dates: %s.", ex.getMessage()); - String correctiveAction = null; - if (ex.getMessage().contains("start date more than end date")) { - message = "Start date more than end date."; - correctiveAction = "Swap dates."; + OffsetDateTime start = OffsetDateTime.parse(getStartDate()); + OffsetDateTime end = OffsetDateTime.parse(getEndDate()); + + if (start.compareTo(end) > 0) { + failureCollector.addFailure("Start date cannot be greater than the end date.", "Swap dates.") + .withConfigProperty(PROPERTY_START_DATE).withConfigProperty(PROPERTY_END_DATE); } - failureCollector.addFailure(message, correctiveAction) - .withConfigProperty(PROPERTY_START_DATE).withConfigProperty(PROPERTY_END_DATE); } catch (DateTimeParseException ex) { - failureCollector.addFailure("Failed to parse one of dates.", - "Correct dates to ISO 8601 format.") - .withConfigProperty(PROPERTY_START_DATE).withConfigProperty(PROPERTY_END_DATE); + // silently ignore parsing exceptions, we already pushed messages for malformed dates } } } @@ -160,14 +175,12 @@ void validateReportType(FailureCollector failureCollector) { void validateSecrets(FailureCollector failureCollector) { if (!containsMacro(PROPERTY_CLIENT_ID) && Strings.isNullOrEmpty(getClientId())) { - failureCollector.addFailure("Client ID is null or empty.", - "Set Client ID to non empty string.") + failureCollector.addFailure("Client ID is empty.", null) .withConfigProperty(PROPERTY_CLIENT_ID); } if (!containsMacro(PROPERTY_CLIENT_SECRET) && Strings.isNullOrEmpty(getClientSecret())) { - failureCollector.addFailure("Client Secret is null or empty.", - "Set Client Secret to non empty string.") + failureCollector.addFailure("Client Secret is empty.", null) .withConfigProperty(PROPERTY_CLIENT_SECRET); } } @@ -178,8 +191,7 @@ void validateMarketoEndpoint(FailureCollector failureCollector) { new URL(getRestApiEndpoint()); } catch (MalformedURLException e) { failureCollector - .addFailure(String.format("Malformed Marketo Rest API endpoint URL '%s'.", getRestApiEndpoint()), - "Change URL to valid.") + .addFailure(String.format("Malformed Marketo Rest API endpoint URL '%s'.", getRestApiEndpoint()), null) .withConfigProperty(PROPERTY_REST_API_ENDPOINT); } } diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/ReportType.java b/src/main/java/io/cdap/plugin/marketo/source/batch/ReportType.java index 1f19f6a..13193f0 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/ReportType.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/ReportType.java @@ -35,6 +35,6 @@ public static ReportType fromString(String reportType) { return rt; } } - throw new IllegalArgumentException("unknown report type: " + reportType); + throw new IllegalArgumentException("Unknown report type: " + reportType); } } diff --git a/widgets/MarketoReportingPlugin-batchsource.json b/widgets/MarketoReportingPlugin-batchsource.json index 96bf773..e5390ba 100644 --- a/widgets/MarketoReportingPlugin-batchsource.json +++ b/widgets/MarketoReportingPlugin-batchsource.json @@ -52,12 +52,18 @@ { "widget-type": "textbox", "label": "Start Date", - "name": "startDate" + "name": "startDate", + "widget-attributes": { + "placeholder": "Start date in ISO 8601 format(1997-07-16T19:20:30+01:00)" + } }, { "widget-type": "textbox", "label": "End Date", - "name": "endDate" + "name": "endDate", + "widget-attributes": { + "placeholder": "End date in ISO 8601 format(1997-07-16T19:20:30+01:00)" + } } ] } From 701b466e1e9892dd16b03327e33de009745c57ec Mon Sep 17 00:00:00 2001 From: Yevhenii Chekanskyi Date: Thu, 28 Nov 2019 17:47:28 +0200 Subject: [PATCH 11/12] PLUGIN-75 Marketo Plugin - extend ignore. --- .gitignore | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 34e1547..8731f78 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,54 @@ -target + +*.class +.*.swp +.beamer +# Package Files # +*.jar +*.war +*.ear +*.versionsBackup + +# Intellij Files & Dir # +*.iml +*.ipr +*.iws +atlassian-ide-plugin.xml +out/ +.DS_Store +./lib/ .idea -*.iml \ No newline at end of file + +# Gradle Files & Dir # +build/ +.gradle/ +.stickyStorage +.build/ +target/ + +# Node log +npm-*.log +logs/ + +# Singlenode and test data files. +/templates/ +/data/ +/data-fabric-tests/data/ + +# ANTLR4 +/core/gen +*.tokens +DirectivesLexer.java +DirectivesParser.java +DirectivesBaseListener.java +DirectivesBaseVisitor.java +DirectivesListener.java +DirectivesVisitor.java + +# generated by docs build +*.py + +# Remove release.properties +release.properties + +# Remove dev directory. +dev \ No newline at end of file From bae9c92af3af03152d155a74c034ac8daa61fb4f Mon Sep 17 00:00:00 2001 From: Yevhenii Chekanskyi Date: Wed, 4 Dec 2019 17:57:47 +0200 Subject: [PATCH 12/12] PLUGIN-75 Marketo Plugin - handle possible race condition. --- pom.xml | 6 ++- .../plugin/marketo/common/api/Marketo.java | 43 ++++++++++++------- .../marketo/common/api/MarketoHttp.java | 19 +++++++- .../common/api/TooManyJobsException.java | 23 ++++++++++ .../common/api/job/AbstractBulkExportJob.java | 8 +++- .../source/batch/MarketoRecordReader.java | 10 +++-- 6 files changed, 86 insertions(+), 23 deletions(-) create mode 100644 src/main/java/io/cdap/plugin/marketo/common/api/TooManyJobsException.java diff --git a/pom.xml b/pom.xml index e8a095b..f7a867b 100644 --- a/pom.xml +++ b/pom.xml @@ -274,7 +274,11 @@ commons-io 2.6 - + + org.awaitility + awaitility + 4.0.1 + diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java b/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java index 809702c..4ccd799 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/Marketo.java @@ -29,6 +29,8 @@ import io.cdap.plugin.marketo.common.api.entities.leads.LeadsExportResponse; import io.cdap.plugin.marketo.common.api.job.ActivitiesExportJob; import io.cdap.plugin.marketo.common.api.job.LeadsExportJob; +import org.awaitility.Awaitility; +import org.awaitility.core.ConditionTimeoutException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,6 +39,7 @@ import java.util.List; import java.util.Spliterator; import java.util.Spliterators; +import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -51,6 +54,14 @@ public class Marketo extends MarketoHttp { "activityTypeId", "campaignId", "primaryAttributeValueId", "primaryAttributeValue", "attributes"); + /** + * Job queue will be checked every 60 seconds. + */ + private static final long JOB_QUEUE_POLL_INTERVAL = 60; + /** + * Wait for 10 seconds before trying to enqueue job, this will minimize chance of race condition. + */ + private static final long JOB_QUEUE_POLL_DELAY = 10; public Marketo(String marketoEndpoint, String clientId, String clientSecret) { super(marketoEndpoint, clientId, clientSecret); @@ -104,22 +115,17 @@ public ActivitiesExport activitiesExportJobStatus(String jobId) { * @param action action to execute once slot is available * @param timeoutSeconds timeout in seconds */ - public void onBulkExtractQueueAvailable(Runnable action, long timeoutSeconds) { - long timeoutMillis = TimeUnit.SECONDS.toMillis(timeoutSeconds); - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() - startTime < timeoutMillis) { - if (canEnqueueJob()) { - action.run(); - return; - } else { - try { - Thread.sleep(TimeUnit.SECONDS.toMillis(60)); - } catch (InterruptedException e) { - throw new RuntimeException("Failed to get slot in bulk export queue - interrupted"); - } - } + public void onBulkExtractQueueAvailable(Callable action, long timeoutSeconds) { + try { + Awaitility.given() + .ignoreException(TooManyJobsException.class) // ignore exception in case another reader took our slot + .atMost(timeoutSeconds, TimeUnit.SECONDS) + .pollInterval(JOB_QUEUE_POLL_INTERVAL, TimeUnit.SECONDS) + .pollDelay(JOB_QUEUE_POLL_DELAY, TimeUnit.SECONDS) + .until(action); + } catch (ConditionTimeoutException ex) { + throw new RuntimeException("Failed to get slot in bulk export queue due to timeout"); } - throw new RuntimeException("Failed to get slot in bulk export queue due to timeout"); } public static LeadsExportResponse streamToLeadsExport(InputStream inputStream) { @@ -130,7 +136,12 @@ public static ActivitiesExportResponse streamToActivitiesExport(InputStream inpu return Helpers.streamToObject(inputStream, ActivitiesExportResponse.class); } - private boolean canEnqueueJob() { + /** + * Check if job can be enqueued. + * + * @return true, if job can be enqueued + */ + public boolean canEnqueueJob() { LeadsExportResponse leadsExportResponseJobs = validatedGet(Urls.BULK_EXPORT_LEADS_LIST, ImmutableMap.of("status", "queued,processing"), Marketo::streamToLeadsExport diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java index c4e7cf1..44c8698 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/MarketoHttp.java @@ -107,6 +107,7 @@ public T validatedPost(String queryUrl, Map T retryableValidate(String logUri, Supplier tryQuery) { T result = tryQuery.get(); // check for expired token @@ -139,11 +140,27 @@ private T retryableValidate(String logUri, Supplier msg = msg + " - " + errors; LOG.error(msg); } - throw new RuntimeException(msg); + throw mapErrorsToException(result.getErrors(), msg); } return result; } + private RuntimeException mapErrorsToException(List errors, String defaultMessage) { + if (errors.size() == 1) { + Error e = errors.get(0); + String message = e.getMessage(); + if (e.getCode() == 1029 && message != null && message.contains("many jobs")) { + return new TooManyJobsException(); + } else { + // this error don't require specific handling + return new RuntimeException(defaultMessage); + } + } else { + // something outstanding happened and we have more than one error, we can't handle this in specific way + return new RuntimeException(defaultMessage); + } + } + public T get(URI uri, Function deserializer) { try (CloseableHttpClient httpClient = HttpClients.createDefault()) { HttpGet request = new HttpGet(uri); diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/TooManyJobsException.java b/src/main/java/io/cdap/plugin/marketo/common/api/TooManyJobsException.java new file mode 100644 index 0000000..ca23321 --- /dev/null +++ b/src/main/java/io/cdap/plugin/marketo/common/api/TooManyJobsException.java @@ -0,0 +1,23 @@ +/* + * Copyright © 2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.marketo.common.api; + +/** + * Exception thrown if too many jobs already queued. + */ +public class TooManyJobsException extends RuntimeException { +} diff --git a/src/main/java/io/cdap/plugin/marketo/common/api/job/AbstractBulkExportJob.java b/src/main/java/io/cdap/plugin/marketo/common/api/job/AbstractBulkExportJob.java index f58109a..de21ca8 100644 --- a/src/main/java/io/cdap/plugin/marketo/common/api/job/AbstractBulkExportJob.java +++ b/src/main/java/io/cdap/plugin/marketo/common/api/job/AbstractBulkExportJob.java @@ -97,12 +97,16 @@ public void waitCompletion() { } } - public void enqueue() { + public boolean enqueue() { if (!getStateStatus(getLastState()).equals(ENQUEUE_ABLE_STATUS)) { throw new IllegalStateException("Job must be in Created status before enqueuing, but was in " + getStateStatus(getLastState())); } + if (!marketo.canEnqueueJob()) { + return false; + } + T newState = enqueueImpl(); logStatusChange(getStateStatus(getLastState()), getStateStatus(newState)); @@ -113,6 +117,8 @@ public void enqueue() { } lastState = newState; + + return true; } private void logStatusChange(String oldStatus, String newStatus) { diff --git a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java index f6b306b..4af14f6 100644 --- a/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java +++ b/src/main/java/io/cdap/plugin/marketo/source/batch/MarketoRecordReader.java @@ -49,6 +49,10 @@ public class MarketoRecordReader extends RecordReader> { private static final Logger LOG = LoggerFactory.getLogger(MarketoRecordReader.class); private static final Gson GSON = new GsonBuilder().create(); + /** + * Wait for 25 minutes for available slot in queue. + */ + private static final long JOB_ENQUEUE_TIMEOUT = 25 * 60; private Map current = null; private Iterator iterator = null; private String beginDate; @@ -94,14 +98,12 @@ public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptCont LOG.info("BULK EXPORT JOB - job '{}' has date range '{}'", job.getJobId(), dateRange); - // TODO handle possible concurrent issues here, another mapper can take our slot - // wait for 10 minutes for available slot and enqueue job - marketo.onBulkExtractQueueAvailable(job::enqueue, 60 * 10); + // wait for 25 minutes for available slot and enqueue job + marketo.onBulkExtractQueueAvailable(job::enqueue, JOB_ENQUEUE_TIMEOUT); job.waitCompletion(); String data = job.getFile(); - // TODO stream here CSVParser parser = CSVFormat.DEFAULT.withHeader().parse(new StringReader(data)); iterator = parser.iterator(); }