TDL-26228 New Stream addition

CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## 2.7.0
+  * New Streams added (Describe Leads,Program Tags, Tag Types) [#95](https://github.com/singer-io/tap-marketo/pull/95)
+
 ## 2.6.1
   * Log Corona warning only if the leads and activities_* stream are selected [#94](https://github.com/singer-io/tap-marketo/pull/94)
 

setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup
 
 setup(name='tap-marketo',
-      version='2.6.1',
+      version='2.7.0',
       description='Singer.io tap for extracting data from the Marketo API',
       author='Stitch',
       url='http://singer.io',
@@ -32,6 +32,9 @@
               "activity_types.json",
               "campaigns.json",
               "programs.json",
+              "leads_describe.json",
+              "program_tags.json",
+              "tag_types.json"
           ]
       },
       include_package_data=True,

tap_marketo/discover.py

@@ -21,6 +21,9 @@
 LISTS_AUTOMATIC_INCLUSION = frozenset(["id", "name", "createdAt", "updatedAt"])
 PROGRAMS_AUTOMATIC_INCLUSION = frozenset(["id", "createdAt", "updatedAt"])
 CAMPAIGNS_AUTOMATIC_INCLUSION = frozenset(["id", "createdAt", "updatedAt"])
+LEADS_DESCRIBE_AUTOMATIC_INCLUSION = frozenset(["id","displayName"])
+PROGRAM_TAGS_AUTOMATIC_INCLUSION = frozenset(["tagType","program_id"])
+TAG_TYPES_AUTOMATIC_INCLUSION = frozenset(["tagType"])
 
 LEAD_REQUIRED_FIELDS = frozenset(["id", "updatedAt", "createdAt"])
 
@@ -82,6 +85,7 @@ def get_activity_type_stream(activity):
         "activityDate": {"type": ["null", "string"], "format": "date-time"},
         "activityTypeId": {"type": ["null", "integer"]},
         "campaignId": {"type": ["null", "integer"]},
+        "actionResult": {"type": ["null", "string"]},
     }
 
     for prop in properties:
@@ -190,6 +194,9 @@ def discover_catalog(name, automatic_inclusion, **kwargs):
 
     with open(path, "r") as f:
         discovered_schema = json.load(f)
+        repl_method = discovered_schema.get("replication_method", None)
+        key_props = discovered_schema.get("key_properties", None)
+
 
         for field in discovered_schema["schema"]["properties"]:
             if field in automatic_inclusion:
@@ -202,8 +209,12 @@ def discover_catalog(name, automatic_inclusion, **kwargs):
         if stream_automatic_inclusion:
             mdata = metadata.write(mdata, (), 'inclusion', 'automatic')
 
-        # The steams using discover_catalog all use "id" as the key_properties
-        mdata = metadata.write(mdata, (), 'table-key-properties', ['id'])
+        if not key_props:
+            key_props = []
+        mdata = metadata.write(mdata, (), 'table-key-properties', key_props)
+
+        if repl_method:
+            mdata = metadata.write(mdata, (), 'forced-replication-method', repl_method)
 
         discovered_schema["metadata"] = metadata.to_list(mdata)
         return discovered_schema
@@ -217,5 +228,8 @@ def discover(client):
     streams.append(discover_catalog("campaigns", CAMPAIGNS_AUTOMATIC_INCLUSION))
     streams.append(discover_catalog("lists", LISTS_AUTOMATIC_INCLUSION))
     streams.append(discover_catalog("programs", PROGRAMS_AUTOMATIC_INCLUSION))
+    streams.append(discover_catalog("leads_describe", LEADS_DESCRIBE_AUTOMATIC_INCLUSION))
+    streams.append(discover_catalog("program_tags", PROGRAM_TAGS_AUTOMATIC_INCLUSION))
+    streams.append(discover_catalog("tag_types", TAG_TYPES_AUTOMATIC_INCLUSION))
     json.dump({"streams": streams}, sys.stdout, indent=2)
     singer.log_info("Finished discover")

tap_marketo/schemas/leads_describe.json

@@ -0,0 +1,45 @@
+{
+    "tap_stream_id": "leads_describe",
+    "stream": "leads_describe",
+    "key_properties": ["id"],
+    "schema": {
+        "type": "object",
+        "additionalProperties": false,
+        "properties": {
+            "id": {
+                "type": "integer"
+            },
+            "displayName": {
+                "type": ["null", "string"]
+            },
+            "dataType": {
+                "type": ["null", "string"]
+            },
+            "length": {
+                "type": ["null", "integer"]
+            },
+            "soap": {
+                "type": ["null", "object"],
+                "properties": {
+                    "name": {
+                        "type": ["null", "string"]
+                    },
+                    "readOnly": {
+                        "type": ["null", "boolean"]
+                    }
+                }
+            },
+            "rest": {
+                "type": ["null", "object"],
+                "properties": {
+                    "name": {
+                        "type": ["null", "string"]
+                    },
+                    "readOnly": {
+                        "type": ["null", "boolean"]
+                    }
+                }
+            }
+        }
+    }
+}

tap_marketo/schemas/program_tags.json

@@ -0,0 +1,21 @@
+{
+  "tap_stream_id": "program_tags",
+  "stream": "program_tags",
+  "replication_method": "FULL_TABLE",
+  "key_properties": [],
+  "schema": {
+    "type": "object",
+    "additionalProperties": false,
+    "properties": {
+      "program_id": {
+        "type": "integer"
+      },
+      "tagType": {
+        "type": [ "null", "string"]
+      },
+      "tagValue": {
+        "type": [ "null", "string"]
+      }
+    }
+  }
+}

tap_marketo/schemas/programs.json

@@ -18,7 +18,13 @@
         "format": "date-time"
       },
       "name": {
-        "type": "string"
+        "type": ["null", "string"]
+      },
+      "sfdcName": {
+       "type": ["null", "string"]
+      },
+      "sfdcId": {
+       "type": ["null", "string"]
       },
       "description": {
         "type": ["null", "string"]

tap_marketo/schemas/tag_types.json

@@ -0,0 +1,21 @@
+{
+    "tap_stream_id": "tag_types",
+    "stream": "tag_types",
+    "key_properties": [],
+    "replication_method": "FULL_TABLE",
+    "schema": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "applicableProgramTypes": {
+          "type": ["null", "string"]
+        },
+        "tagType": {
+          "type": ["null", "string"]
+        },
+        "required": {
+          "type": ["boolean", "null"]
+        }
+      }
+    }
+  }