From 199d51539da7c4ab25a4e83988eadb7895c5323b Mon Sep 17 00:00:00 2001 From: pnadolny13 Date: Mon, 23 Oct 2023 12:08:35 +0000 Subject: [PATCH] Refresh plugin metadata 2023-10-23 --- .../extractors/tap-cassandra/datarts-tech.yml | 6 +- .../meltano/extractors/tap-db2/danielptv.yml | 1 + .../extractors/tap-dbt/meltanolabs.yml | 2 +- .../meltano/extractors/tap-gmail/kgpayne.yml | 2 +- .../extractors/tap-gmail/meltanolabs.yml | 2 +- .../tap-google-analytics/connorflyn.yml | 87 ++++ .../tap-google-analytics/meltanolabs.yml | 141 ++----- .../tap-google-analytics/sehnem.yml | 82 ++++ .../extractors/tap-gorgias/brooklyn-data.yml | 2 +- .../extractors/tap-hubspot/spacecowboy.yml | 2 +- .../extractors/tap-jira/meltanolabs.yml | 23 +- .../extractors/tap-klaviyo/meltanolabs.yml | 2 +- .../extractors/tap-mailchimp/acarter24.yml | 2 +- .../extractors/tap-postgres/meltanolabs.yml | 13 + .../extractors/tap-rest-api-msdk/widen.yml | 2 +- .../tap-saasoptics/datarts-tech.yml | 2 +- .../extractors/tap-salesforce/hotgluexyz.yml | 64 +++ .../extractors/tap-talon-one/gthesheep.yml | 5 + .../extractors/tap-totango/edsoncezar16.yml | 6 +- .../extractors/tap-tradera/kgpayne.yml | 4 +- .../loaders/target-postgres/meltanolabs.yml | 15 + _data/meltano/loaders/target-s3/crowemi.yml | 386 ++++++++++++++++-- _data/meltano/loaders/target-s3/jkausti.yml | 72 ++++ .../loaders/target-timeplus/timeplus.yml | 2 +- 24 files changed, 742 insertions(+), 183 deletions(-) create mode 100644 _data/meltano/extractors/tap-google-analytics/connorflyn.yml create mode 100644 _data/meltano/extractors/tap-google-analytics/sehnem.yml create mode 100644 _data/meltano/extractors/tap-salesforce/hotgluexyz.yml create mode 100644 _data/meltano/loaders/target-s3/jkausti.yml diff --git a/_data/meltano/extractors/tap-cassandra/datarts-tech.yml b/_data/meltano/extractors/tap-cassandra/datarts-tech.yml index d44578128..987d9f3ee 100644 --- a/_data/meltano/extractors/tap-cassandra/datarts-tech.yml +++ b/_data/meltano/extractors/tap-cassandra/datarts-tech.yml @@ -65,7 +65,7 @@ settings: name: keyspace - description: The local_dc parameter should be the name of the datacenter. kind: string - label: Local Datacenter + label: Local Dc name: local_dc - description: Should be a total number of attempts to be made before giving up. kind: integer @@ -76,7 +76,7 @@ settings: kind: password label: Password name: password -- description: The server-side port to open connections to. Defaults to 9042.. +- description: The server-side port to open connections to. Defaults to 9042. . kind: integer label: Port name: port @@ -91,7 +91,7 @@ settings: label: Reconnect Delay name: reconnect_delay value: 60 -- description: Request timeout used when not overridden in Session.execute(). +- description: Request timeout used when not overridden in Session. Execute(). kind: integer label: Request Timeout name: request_timeout diff --git a/_data/meltano/extractors/tap-db2/danielptv.yml b/_data/meltano/extractors/tap-db2/danielptv.yml index 24338a7e7..3cd489024 100644 --- a/_data/meltano/extractors/tap-db2/danielptv.yml +++ b/_data/meltano/extractors/tap-db2/danielptv.yml @@ -2,6 +2,7 @@ capabilities: - about - catalog - discover +- state - stream-maps description: IBM DB2 Relational Database domain_url: https://www.ibm.com/products/db2 diff --git a/_data/meltano/extractors/tap-dbt/meltanolabs.yml b/_data/meltano/extractors/tap-dbt/meltanolabs.yml index 88a319b55..efa29a5b1 100644 --- a/_data/meltano/extractors/tap-dbt/meltanolabs.yml +++ b/_data/meltano/extractors/tap-dbt/meltanolabs.yml @@ -84,7 +84,7 @@ settings: kind: string label: User Agent name: user_agent - value: tap-dbt/0.8.3.dev9+eff3995 Singer tap for the dbt Cloud API. + value: tap-dbt/0.9.1.dev1+533456b Singer tap for the dbt Cloud API. settings_group_validation: - - account_ids - api_key diff --git a/_data/meltano/extractors/tap-gmail/kgpayne.yml b/_data/meltano/extractors/tap-gmail/kgpayne.yml index 283ff6f41..8c04b989a 100644 --- a/_data/meltano/extractors/tap-gmail/kgpayne.yml +++ b/_data/meltano/extractors/tap-gmail/kgpayne.yml @@ -36,7 +36,7 @@ settings: value: false - description: Only return messages matching the specified query. Supports the same query format as the Gmail search box. For example, "from:someuser@example.com - rfc822msgid: is:unread". Parameter cannot be used when + rfc822msgid: is:unread". Parameter cannot be used when accessing the api using the gmail. Metadata scope. Https://developers. Google. Com/gmail/api/reference/rest/v1/users. Messages/list#query-parameters kind: string diff --git a/_data/meltano/extractors/tap-gmail/meltanolabs.yml b/_data/meltano/extractors/tap-gmail/meltanolabs.yml index 04d579362..eff333258 100644 --- a/_data/meltano/extractors/tap-gmail/meltanolabs.yml +++ b/_data/meltano/extractors/tap-gmail/meltanolabs.yml @@ -35,7 +35,7 @@ settings: value: false - description: Only return messages matching the specified query. Supports the same query format as the Gmail search box. For example, "from:someuser@example.com - rfc822msgid: is:unread". Parameter cannot be used when + rfc822msgid: is:unread". Parameter cannot be used when accessing the api using the gmail. Metadata scope. Https://developers. Google. Com/gmail/api/reference/rest/v1/users. Messages/list#query-parameters kind: string diff --git a/_data/meltano/extractors/tap-google-analytics/connorflyn.yml b/_data/meltano/extractors/tap-google-analytics/connorflyn.yml new file mode 100644 index 000000000..783bc2eb4 --- /dev/null +++ b/_data/meltano/extractors/tap-google-analytics/connorflyn.yml @@ -0,0 +1,87 @@ +capabilities: +- about +- catalog +- discover +- schema-flattening +- state +- stream-maps +description: App and website analytics platform hosted by Google (GA4) +domain_url: https://developers.google.com/analytics/devguides/reporting/data/v1 +executable: tap-google-analytics +hidden: true +keywords: +- meltano_sdk +label: Google Analytics (GA4) +logo_url: /assets/logos/extractors/ga4.png +maintenance_status: active +name: tap-ga4 +namespace: tap_ga4 +next_steps: '' +pip_url: git+https://github.com/connorflyn/tap-google-analytics-v4.git +quality: bronze +repo: https://github.com/connorflyn/tap-google-analytics-v4 +settings: +- description: Google Analytics Client Secrets Dictionary + kind: object + label: Client Secrets + name: client_secrets +- description: The last record date to sync + kind: date_iso8601 + label: End Date + name: end_date +- description: "'True' to enable schema flattening and automatically expand nested + properties." + kind: boolean + label: Flattening Enabled + name: flattening_enabled +- description: The max depth to flatten schemas. + kind: integer + label: Flattening Max Depth + name: flattening_max_depth +- description: File Path to Google Analytics Client Secrets + kind: password + label: Key File Location + name: key_file_location +- description: Google Analytics Access Token + kind: password + label: OAuth Credentials Access Token + name: oauth_credentials.access_token +- description: Google Analytics Client ID + kind: password + label: OAuth Credentials Client ID + name: oauth_credentials.client_id +- description: Google Analytics Client Secret + kind: password + label: OAuth Credentials Client Secret + name: oauth_credentials.client_secret +- description: Google Analytics Refresh Token + kind: password + label: OAuth Credentials Refresh Token + name: oauth_credentials.refresh_token +- description: Google Analytics Property ID + kind: password + label: Property ID + name: property_id +- description: Google Analytics Reports Definition + kind: string + label: Reports + name: reports +- description: The earliest record date to sync + kind: date_iso8601 + label: Start Date + name: start_date +- description: User-defined config values to be used within map expressions. + kind: object + label: Stream Map Config + name: stream_map_config +- description: Config object for stream maps capability. For more information check + out [Stream Maps](https://sdk.meltano.com/en/latest/stream_maps.html). + kind: object + label: Stream Maps + name: stream_maps +settings_group_validation: +- - property_id + - start_date +settings_preamble: '' +usage: '' +variant: connorflyn diff --git a/_data/meltano/extractors/tap-google-analytics/meltanolabs.yml b/_data/meltano/extractors/tap-google-analytics/meltanolabs.yml index a19d7426c..535ebea94 100644 --- a/_data/meltano/extractors/tap-google-analytics/meltanolabs.yml +++ b/_data/meltano/extractors/tap-google-analytics/meltanolabs.yml @@ -5,26 +5,26 @@ capabilities: - schema-flattening - state - stream-maps -description: App and website analytics platform hosted by Google -domain_url: https://developers.google.com/analytics/devguides/reporting/core/v4/ +description: App and website analytics platform hosted by Google (GA4) +domain_url: https://developers.google.com/analytics/devguides/reporting/data/v1 +executable: tap-google-analytics keywords: - api - meltano_sdk -label: Google Analytics (Universal Analytics API - Deprecated) -logo_url: /assets/logos/extractors/google-analytics.png +label: Google Analytics (GA4) +logo_url: /assets/logos/extractors/ga4.png maintenance_status: active -name: tap-google-analytics -namespace: tap_google_analytics -pip_url: git+https://github.com/MeltanoLabs/tap-google-analytics.git@main_deprecated_ua +name: tap-ga4 +namespace: tap_ga4 +next_steps: '' +pip_url: git+https://github.com/MeltanoLabs/tap-google-analytics.git quality: gold repo: https://github.com/MeltanoLabs/tap-google-analytics settings: -- description: | - Follow the above steps for [Key File Location](#key-file-location) but instead of providing a path you can provide the serialized json directly. This can be useful for ephemeral runtime environments where its easier to provide an environment variable instead of a file. +- description: Google Analytics Client Secrets Dictionary kind: object label: Client Secrets name: client_secrets - placeholder: Ex. client_secrets.json - description: Date up to when historical data will be extracted. kind: date_iso8601 label: End Date @@ -38,77 +38,17 @@ settings: kind: integer label: Flattening Max Depth name: flattening_max_depth -- description: | - #### How to get - - Follow the steps below if you don't already have a valid `client_secrets.json` to upload. The process below can take over 10 minutes, but it's a one-time setup that's well worth it. - - This extractor supports service account based authorization, where an administrator manually creates a service account with the appropriate permissions to view the account, property, and view you wish to fetch data from. - - To access your Google Analytics data, the "Analytics Reporting API" and "Analytics API" both need to be enabled. These need to be enabled for a project inside the same organization as your Google Analytics account. - - ##### Step 1: Creating Service Account Credentials - - As a first step, you need to create a new project in Google Cloud Platform or use an existing one: - - 1. Sign in to the Google Account you are using for managing Google Analytics (you must have Manage Users permission at the account, property, or view level). - - 2. Open the [Service accounts page](https://console.developers.google.com/iam-admin/serviceaccounts). If prompted, select a project or create a new one to use for accessing Google Analytics. - - ![Screenshot of Google Service Accounts page](/assets/images/tap-google-analytics/02-ga-service-account-configuration-create-new-account.png) - - 3. Click "Create service account" - - In the Create service account window, type a name for the service account, and click `Create`. - - We do not need to provide any additional permissions for this account, so click `Continue` in the `Service account permissions` configuration page. - - We also do not need to grant access to any users for this service account, as we only need the key. - - ![Screenshot of Google Service Account Configuration for new Account](/assets/images/tap-google-analytics/02-ga-service-account-configuration-new-account.png) - - Click `Create Key`, select `JSON` as the key type and create a new private key. Then click `Save` and store it locally as `client_secrets.json`. - - Meltano will use the private key in this `client_secrets.json` file to connect with the Google Analytics API. - - ##### Step 2: Linking Credentials to Google Analytics - - The newly created service account will have an email address that looks similar to: - - ``` - service-account-name@PROJECT-ID.iam.gserviceaccount.com - ``` - - To grant this service account access to your Google Analytics data, add the email address as a [new user](https://support.google.com/analytics/answer/1009702) to your Google Analytics account, property or view through the "Admin > User Management" page. - - Only the [Read & Analyze permissions](https://support.google.com/analytics/answer/2884495) are needed as Meltano only extracts data to generate reports. - - ![Screenshot of Google Analytics Add User](/assets/images/tap-google-analytics/03-ga-add-user.png) - - ##### Step 3: Enabling the APIs - - 1. Visit the [Google Analytics Reporting API](https://console.developers.google.com/apis/api/analyticsreporting.googleapis.com/overview) dashboard and make sure that the project you used in the previous step is selected. - - Now enable the API using the button at the top, so that the button will say "Disable API" instead: - - ![Screenshot of Google Analytics Reporting API](/assets/images/tap-google-analytics/04-ga-reporting-api.png) - - 2. Next, visit the [Google Analytics API](https://console.developers.google.com/apis/api/analytics.googleapis.com/overview) dashboard, make sure that the project you used in the previous step is selected, and enable this API as well. - - ![Screenshot of Google Analytics API](/assets/images/tap-google-analytics/05-ga-api.png) +- description: File Path to Google Analytics Client Secrets kind: password label: Key File Location name: key_file_location - placeholder: Ex. $MELTANO_PROJECT_ROOT/client_secrets.json - description: Google Analytics Access Token. See https://developers.google.com/analytics/devguides/reporting/core/v4/authorization#OAuth2Authorizing. kind: password label: OAuth Credentials Access Token name: oauth_credentials.access_token -- description: | - See . - - Takes precedence over [Key File Location](#key-file-location) if both are specified. +- description: Google Analytics Client ID. See + https://developers.google.com/analytics/devguides/reporting/core/v4/authorization#OAuth2Authorizing. kind: password label: OAuth Credentials Client ID name: oauth_credentials.client_id @@ -123,9 +63,12 @@ settings: label: OAuth Credentials Refresh Token name: oauth_credentials.refresh_token - description: | - Project-relative path to JSON file with the definition of the reports to be generated. + Google Analytics Reports Definition. + The tap uses the [default reports definition](https://github.com/MeltanoLabs/tap-google-analytics/blob/main/tap_google_analytics/defaults/default_report_definition.json) + if this field is not provided. + A project-relative path to JSON file with the definition of the reports to be generated. - See for valid dimensions and metrics. + See for valid dimensions and metrics. The JSON structure expected is as follows: @@ -156,25 +99,25 @@ settings: { "name" : "users_per_day", "dimensions" : [ - "ga:date" + "date" ], "metrics" : [ - "ga:users", - "ga:newUsers" + "newUsers", + "active1DayUsers" ] }, { "name" : "sessions_per_country_day", "dimensions" : [ - "ga:date", - "ga:country" + "date", + "country" ], "metrics" : [ - "ga:sessions", - "ga:sessionsPerUser", - "ga:avgSessionDuration" + "sessions", + "sessionsPerUser", + "avgSessionDuration" ] } ] @@ -198,39 +141,23 @@ settings: kind: object label: Stream Maps name: stream_maps -- description: | - The ID for the view to fetch data from. - - #### How to get - - To get your View ID: - - 1. Visit Google Analytics: - 2. Log in if you haven't already. - 3. Open the account/property/view selector in the top left corner - - ![Screenshot of closed account selector](/assets/images/tap-google-analytics/account-selector-closed.png) - - 3. Select the account, property, and view that you would like to connect with Meltano - - ![Screenshot of open account selector](/assets/images/tap-google-analytics/account-selector-open.png) - - 4. You will see the View ID displayed inside the selector below the name of the view (e.g. "All Web Site Data"): `188274549` +- description: Google Analytics View ID kind: password label: View ID name: view_id - placeholder: Ex. 198343027 settings_group_validation: -- - key_file_location - - start_date - - view_id - - client_secrets + - property_id + - start_date +- - key_file_location + - property_id - start_date - - view_id - - oauth_credentials.access_token - oauth_credentials.client_id - oauth_credentials.client_secret - oauth_credentials.refresh_token + - property_id - start_date - - view_id +settings_preamble: '' +usage: '' variant: meltanolabs diff --git a/_data/meltano/extractors/tap-google-analytics/sehnem.yml b/_data/meltano/extractors/tap-google-analytics/sehnem.yml new file mode 100644 index 000000000..e8a3a5fc9 --- /dev/null +++ b/_data/meltano/extractors/tap-google-analytics/sehnem.yml @@ -0,0 +1,82 @@ +capabilities: +- about +- catalog +- discover +- schema-flattening +- state +- stream-maps +description: App and website analytics platform hosted by Google (GA4) +domain_url: https://developers.google.com/analytics/devguides/reporting/data/v1 +executable: tap-google-analytics +hidden: true +keywords: +- meltano_sdk +label: Google Analytics (GA4) +logo_url: /assets/logos/extractors/ga4.png +maintenance_status: active +name: tap-ga4 +namespace: tap_ga4 +next_steps: '' +pip_url: git+https://github.com/sehnem/tap-google-analytics.git +quality: silver +repo: https://github.com/sehnem/tap-google-analytics +settings: +- description: Google Analytics Access Token + kind: password + label: Access Token + name: access_token +- description: Google Analytics Client ID + kind: password + label: Client ID + name: client_id +- description: Google Analytics Client Secret + kind: password + label: Client Secret + name: client_secret +- description: The last record date to sync + kind: date_iso8601 + label: End Date + name: end_date +- description: "'True' to enable schema flattening and automatically expand nested + properties." + kind: boolean + label: Flattening Enabled + name: flattening_enabled +- description: The max depth to flatten schemas. + kind: integer + label: Flattening Max Depth + name: flattening_max_depth +- description: Google Analytics Property ID + kind: password + label: Property ID + name: property_id +- description: Google Analytics Refresh Token + kind: password + label: Refresh Token + name: refresh_token +- description: Google Analytics Reports Definition + kind: string + label: Reports + name: reports +- description: The earliest record date to sync + kind: date_iso8601 + label: Start Date + name: start_date +- description: User-defined config values to be used within map expressions. + kind: object + label: Stream Map Config + name: stream_map_config +- description: Config object for stream maps capability. For more information check + out [Stream Maps](https://sdk.meltano.com/en/latest/stream_maps.html). + kind: object + label: Stream Maps + name: stream_maps +settings_group_validation: +- - client_id + - client_secret + - property_id + - refresh_token + - start_date +settings_preamble: '' +usage: '' +variant: sehnem diff --git a/_data/meltano/extractors/tap-gorgias/brooklyn-data.yml b/_data/meltano/extractors/tap-gorgias/brooklyn-data.yml index 33393583d..7d5a10398 100644 --- a/_data/meltano/extractors/tap-gorgias/brooklyn-data.yml +++ b/_data/meltano/extractors/tap-gorgias/brooklyn-data.yml @@ -54,7 +54,7 @@ settings: kind: object label: Stream Maps name: stream_maps -- description: Subdomain (.gorgias.com) +- description: Subdomain (. Gorgias. Com) kind: string label: Subdomain name: subdomain diff --git a/_data/meltano/extractors/tap-hubspot/spacecowboy.yml b/_data/meltano/extractors/tap-hubspot/spacecowboy.yml index a560d7d94..f4e2051fd 100644 --- a/_data/meltano/extractors/tap-hubspot/spacecowboy.yml +++ b/_data/meltano/extractors/tap-hubspot/spacecowboy.yml @@ -53,7 +53,7 @@ settings: name: flattening_max_depth - description: HubSpot private app token kind: password - label: HubSpot API Key + label: Hapikey name: hapikey - description: Set to True to avoid using the search API - implies full table replication kind: boolean diff --git a/_data/meltano/extractors/tap-jira/meltanolabs.yml b/_data/meltano/extractors/tap-jira/meltanolabs.yml index 9d574140f..1ec16424b 100644 --- a/_data/meltano/extractors/tap-jira/meltanolabs.yml +++ b/_data/meltano/extractors/tap-jira/meltanolabs.yml @@ -21,27 +21,10 @@ pip_url: git+https://github.com/MeltanoLabs/tap-jira.git quality: silver repo: https://github.com/MeltanoLabs/tap-jira settings: -- description: The authentication type to use. +- description: '' kind: string - label: Auth Flow - name: auth.flow - options: - - label: OAuth - value: oauth - - label: Password - value: password -- description: The Jira API access token if using OAuth auth flow. - kind: password - label: Auth Access Token - name: auth.access_token -- description: The Jira username if using the user/pass auth flow. - kind: string - label: Auth Username - name: auth.username -- description: The Jira password if using the user/pass auth flow. - kind: password - label: Auth Password - name: auth.password + label: Auth + name: auth - description: Compression format to use for batch files. kind: options label: Batch Config Encoding Compression diff --git a/_data/meltano/extractors/tap-klaviyo/meltanolabs.yml b/_data/meltano/extractors/tap-klaviyo/meltanolabs.yml index dba80c439..1f34b429b 100644 --- a/_data/meltano/extractors/tap-klaviyo/meltanolabs.yml +++ b/_data/meltano/extractors/tap-klaviyo/meltanolabs.yml @@ -34,7 +34,7 @@ settings: kind: integer label: Flattening Max Depth name: flattening_max_depth -- description: Klaviyo API endpoint revision. https://developers.klaviyo.com/en/docs/api_versioning_and_deprecation_policy#versioning +- description: Klaviyo API endpoint revision. Https://developers. Klaviyo. Com/en/docs/api_versioning_and_deprecation_policy#versioning kind: string label: Revision name: revision diff --git a/_data/meltano/extractors/tap-mailchimp/acarter24.yml b/_data/meltano/extractors/tap-mailchimp/acarter24.yml index 9d761ccbf..dfd83ff62 100644 --- a/_data/meltano/extractors/tap-mailchimp/acarter24.yml +++ b/_data/meltano/extractors/tap-mailchimp/acarter24.yml @@ -27,7 +27,7 @@ settings: name: api_key - description: Your Mailchimp DC kind: string - label: Data Center + label: Dc name: dc - description: "'True' to enable schema flattening and automatically expand nested properties." diff --git a/_data/meltano/extractors/tap-postgres/meltanolabs.yml b/_data/meltano/extractors/tap-postgres/meltanolabs.yml index 3b0a2e290..6d50d68a7 100644 --- a/_data/meltano/extractors/tap-postgres/meltanolabs.yml +++ b/_data/meltano/extractors/tap-postgres/meltanolabs.yml @@ -55,6 +55,19 @@ settings: label: Dates As String name: dates_as_string value: false +- description: Replication method to use if there is not a catalog entry to override + this choice. One of `FULL_TABLE`, `INCREMENTAL`, or `LOG_BASED`. + kind: options + label: Default Replication Method + name: default_replication_method + options: + - label: Full Table + value: FULL_TABLE + - label: Incremental + value: INCREMENTAL + - label: Log Based + value: LOG_BASED + value: FULL_TABLE - description: If an array of schema names is provided, the tap will only process the specified Postgres schemas and ignore others. If left blank, the tap automatically determines ALL available Postgres schemas. diff --git a/_data/meltano/extractors/tap-rest-api-msdk/widen.yml b/_data/meltano/extractors/tap-rest-api-msdk/widen.yml index e3fedff3e..fa4e8e9b0 100644 --- a/_data/meltano/extractors/tap-rest-api-msdk/widen.yml +++ b/_data/meltano/extractors/tap-rest-api-msdk/widen.yml @@ -241,7 +241,7 @@ settings: tap will dynamically update the token with either the `start_date`or the last bookmark / state value. A simple template Example for FHIR API's: gt$last_run_date. A more complex example against an Opensearch API, {\"bool\": {\"filter\": [{\"\ - range\": { \"meta.lastUpdated\": { \"gt\": \"$last_run_date\" }}}] }}. Note: Any + range\": { \"meta.lastUpdated\": { \"gt\": \"$last_run_date\" }}}] }} .Note: Any required double quotes in the query template must be escaped." kind: string label: Source Search Query diff --git a/_data/meltano/extractors/tap-saasoptics/datarts-tech.yml b/_data/meltano/extractors/tap-saasoptics/datarts-tech.yml index 5bebd3628..5ad836c2d 100644 --- a/_data/meltano/extractors/tap-saasoptics/datarts-tech.yml +++ b/_data/meltano/extractors/tap-saasoptics/datarts-tech.yml @@ -85,7 +85,7 @@ settings: kind: object label: Stream Maps name: stream_maps -- description: tap-saasoptics . +- description: Tap-saasoptics . kind: string label: User Agent name: user_agent diff --git a/_data/meltano/extractors/tap-salesforce/hotgluexyz.yml b/_data/meltano/extractors/tap-salesforce/hotgluexyz.yml new file mode 100644 index 000000000..a84c68894 --- /dev/null +++ b/_data/meltano/extractors/tap-salesforce/hotgluexyz.yml @@ -0,0 +1,64 @@ +capabilities: +- about +- catalog +- discover +- schema-flattening +- state +- stream-maps +description: Cloud-based commerce solution +domain_url: https://www.salesforce.com/products/commerce-cloud/overview/ +executable: tap-salesforce-commerce +keywords: +- meltano_sdk +label: Salesforce Commerce +logo_url: /assets/logos/extractors/salesforce-commerce.png +maintenance_status: active +name: tap-salesforce-commerce +namespace: tap_salesforce_commerce +next_steps: '' +pip_url: git+https://gitlab.com/hotglue/tap-salesforce-commerce.git +quality: gold +repo: https://gitlab.com/hotglue/tap-salesforce-commerce +settings: +- description: Your Salesforce Client ID. + kind: password + label: Client ID + name: client_id +- description: Your Salesforce Client Secret. + kind: password + label: Client Secret + name: client_secret +- description: "'True' to enable schema flattening and automatically expand nested + properties." + kind: boolean + label: Flattening Enabled + name: flattening_enabled +- description: The max depth to flatten schemas. + kind: integer + label: Flattening Max Depth + name: flattening_max_depth +- description: Your Salesforce Site ID. + kind: password + label: Site ID + name: site_id +- description: "Determines how much historical data will be extracted. Please be aware\n + that the larger the time period and amount of data, the longer the initial extraction\n + can be expected to take." + kind: date_iso8601 + label: Start Date + name: start_date +- description: User-defined config values to be used within map expressions. + kind: object + label: Stream Map Config + name: stream_map_config +- description: Config object for stream maps capability. + kind: object + label: Stream Maps + name: stream_maps +settings_group_validation: +- - client_id + - client_secret + - site_id +settings_preamble: '' +usage: '' +variant: hotgluexyz diff --git a/_data/meltano/extractors/tap-talon-one/gthesheep.yml b/_data/meltano/extractors/tap-talon-one/gthesheep.yml index 4f5f31a57..7e30be8e9 100644 --- a/_data/meltano/extractors/tap-talon-one/gthesheep.yml +++ b/_data/meltano/extractors/tap-talon-one/gthesheep.yml @@ -42,6 +42,11 @@ settings: kind: integer label: Flattening Max Depth name: flattening_max_depth +- description: Page size for pagination + kind: integer + label: Page Size + name: page_size + value: 10000 - description: The earliest record date to sync kind: date_iso8601 label: Start Date diff --git a/_data/meltano/extractors/tap-totango/edsoncezar16.yml b/_data/meltano/extractors/tap-totango/edsoncezar16.yml index a4233424d..8a8afc9e0 100644 --- a/_data/meltano/extractors/tap-totango/edsoncezar16.yml +++ b/_data/meltano/extractors/tap-totango/edsoncezar16.yml @@ -43,7 +43,7 @@ settings: be defined using the count parameter (and limited to 1000). Tip: To page through results, ask for 1000 records (count: 1000). If you receive 1000 records, assume there’s more, in which case you want to pull the next 1000 records (offset: 1000…then - 2000…etc.). Repeat paging until the number of records returned is less than 1000.' + 2000…etc. ). Repeat paging until the number of records returned is less than 1000.' kind: integer label: Accounts Offset name: accounts_offset @@ -75,9 +75,9 @@ settings: label: API URL name: api_url options: - - label: https://api.totango.com + - label: Https://API Totango Com value: https://api.totango.com - - label: 'https://api-eu1.totango.com ' + - label: 'Https://API Eu1 Totango Com ' value: 'https://api-eu1.totango.com ' value: https://api.totango.com - description: The token to authenticate against the API service diff --git a/_data/meltano/extractors/tap-tradera/kgpayne.yml b/_data/meltano/extractors/tap-tradera/kgpayne.yml index 311c57993..72f368bb2 100644 --- a/_data/meltano/extractors/tap-tradera/kgpayne.yml +++ b/_data/meltano/extractors/tap-tradera/kgpayne.yml @@ -19,11 +19,11 @@ pip_url: git+https://github.com/kgpayne/tap-tradera.git quality: silver repo: https://github.com/kgpayne/tap-tradera settings: -- description: The Tradera.com developer app ID +- description: The Tradera. Com developer app ID kind: password label: App ID name: app_id -- description: The Tradera.com developer app key +- description: The Tradera. Com developer app key kind: password label: App Key name: app_key diff --git a/_data/meltano/loaders/target-postgres/meltanolabs.yml b/_data/meltano/loaders/target-postgres/meltanolabs.yml index 5e65630a9..46d01f47a 100644 --- a/_data/meltano/loaders/target-postgres/meltanolabs.yml +++ b/_data/meltano/loaders/target-postgres/meltanolabs.yml @@ -62,6 +62,21 @@ settings: kind: string label: Host name: host +- description: The method to use when loading data into the destination. `append-only` + will always write all input records whether that records already exists or not. + `upsert` will update existing records and insert new records. `overwrite` will + delete all existing records and insert all input records. + kind: options + label: Load Method + name: load_method + options: + - label: Append Only + value: append-only + - label: Upsert + value: upsert + - label: Overwrite + value: overwrite + value: append-only - description: Password used to authenticate. Note if sqlalchemy_url is set this will be ignored. kind: password diff --git a/_data/meltano/loaders/target-s3/crowemi.yml b/_data/meltano/loaders/target-s3/crowemi.yml index a54f98ec8..ad5f22e3e 100644 --- a/_data/meltano/loaders/target-s3/crowemi.yml +++ b/_data/meltano/loaders/target-s3/crowemi.yml @@ -19,6 +19,10 @@ pip_url: git+https://github.com/crowemi/target-s3.git quality: gold repo: https://github.com/crowemi/target-s3 settings: +- description: Add metadata to records. + kind: boolean + label: Add Record Metadata + name: add_record_metadata - description: A flag to append the date to the key filename. kind: boolean label: Append Date To Filename @@ -29,20 +33,162 @@ settings: label: Append Date To Filename Grain name: append_date_to_filename_grain options: - - label: Year - value: year - - label: Month - value: month - - label: Day - value: day - - label: Hour - value: hour - - label: Minute - value: minute - - label: Second - value: second - - label: Microsecond - value: microsecond + - label: D + value: d + - label: I + value: i + - label: C + value: c + - label: T + value: t + - label: ' ' + value: _ + - label: K + value: k + - label: E + value: e + - label: Y + value: y + - label: S + value: s + - label: ( + value: ( + - label: '[' + value: '[' + - label: "'" + value: "'" + - label: Y + value: y + - label: E + value: e + - label: A + value: a + - label: R + value: r + - label: "'" + value: "'" + - label: ',' + value: ',' + - label: ' ' + value: ' ' + - label: "'" + value: "'" + - label: M + value: m + - label: O + value: o + - label: N + value: n + - label: T + value: t + - label: H + value: h + - label: "'" + value: "'" + - label: ',' + value: ',' + - label: ' ' + value: ' ' + - label: "'" + value: "'" + - label: D + value: d + - label: A + value: a + - label: Y + value: y + - label: "'" + value: "'" + - label: ',' + value: ',' + - label: ' ' + value: ' ' + - label: "'" + value: "'" + - label: H + value: h + - label: O + value: o + - label: U + value: u + - label: R + value: r + - label: "'" + value: "'" + - label: ',' + value: ',' + - label: ' ' + value: ' ' + - label: "'" + value: "'" + - label: M + value: m + - label: I + value: i + - label: N + value: n + - label: U + value: u + - label: T + value: t + - label: E + value: e + - label: "'" + value: "'" + - label: ',' + value: ',' + - label: ' ' + value: ' ' + - label: "'" + value: "'" + - label: S + value: s + - label: E + value: e + - label: C + value: c + - label: O + value: o + - label: N + value: n + - label: D + value: d + - label: "'" + value: "'" + - label: ',' + value: ',' + - label: ' ' + value: ' ' + - label: "'" + value: "'" + - label: M + value: m + - label: I + value: i + - label: C + value: c + - label: R + value: r + - label: O + value: o + - label: S + value: s + - label: E + value: e + - label: C + value: c + - label: O + value: o + - label: N + value: n + - label: D + value: d + - label: "'" + value: "'" + - label: ']' + value: ']' + - label: ) + value: ) value: day - description: A flag to append the date to the key prefix. kind: boolean @@ -54,20 +200,162 @@ settings: label: Append Date To Prefix Grain name: append_date_to_prefix_grain options: - - label: Year - value: year - - label: Month - value: month - - label: Day - value: day - - label: Hour - value: hour - - label: Minute - value: minute - - label: Second - value: second - - label: Microsecond - value: microsecond + - label: D + value: d + - label: I + value: i + - label: C + value: c + - label: T + value: t + - label: ' ' + value: _ + - label: K + value: k + - label: E + value: e + - label: Y + value: y + - label: S + value: s + - label: ( + value: ( + - label: '[' + value: '[' + - label: "'" + value: "'" + - label: Y + value: y + - label: E + value: e + - label: A + value: a + - label: R + value: r + - label: "'" + value: "'" + - label: ',' + value: ',' + - label: ' ' + value: ' ' + - label: "'" + value: "'" + - label: M + value: m + - label: O + value: o + - label: N + value: n + - label: T + value: t + - label: H + value: h + - label: "'" + value: "'" + - label: ',' + value: ',' + - label: ' ' + value: ' ' + - label: "'" + value: "'" + - label: D + value: d + - label: A + value: a + - label: Y + value: y + - label: "'" + value: "'" + - label: ',' + value: ',' + - label: ' ' + value: ' ' + - label: "'" + value: "'" + - label: H + value: h + - label: O + value: o + - label: U + value: u + - label: R + value: r + - label: "'" + value: "'" + - label: ',' + value: ',' + - label: ' ' + value: ' ' + - label: "'" + value: "'" + - label: M + value: m + - label: I + value: i + - label: N + value: n + - label: U + value: u + - label: T + value: t + - label: E + value: e + - label: "'" + value: "'" + - label: ',' + value: ',' + - label: ' ' + value: ' ' + - label: "'" + value: "'" + - label: S + value: s + - label: E + value: e + - label: C + value: c + - label: O + value: o + - label: N + value: n + - label: D + value: d + - label: "'" + value: "'" + - label: ',' + value: ',' + - label: ' ' + value: ' ' + - label: "'" + value: "'" + - label: M + value: m + - label: I + value: i + - label: C + value: c + - label: R + value: r + - label: O + value: o + - label: S + value: s + - label: E + value: e + - label: C + value: c + - label: O + value: o + - label: N + value: n + - label: D + value: d + - label: "'" + value: "'" + - label: ']' + value: ']' + - label: ) + value: ) value: day - description: Your AWS access key ID. kind: password @@ -101,16 +389,8 @@ settings: kind: password label: Cloud Provider Cloud Provider Type name: cloud_provider.cloud_provider_type - options: - - label: AWS - value: aws - value: aws -- description: A flag indictating to flatten records. - kind: boolean - label: Flatten Records - name: flatten_records -- description: "'True' to enable schema flattening and automatically expand nested\ - \ properties." +- description: "'True' to enable schema flattening and automatically expand nested + properties." kind: boolean label: Flattening Enabled name: flattening_enabled @@ -118,6 +398,21 @@ settings: kind: integer label: Flattening Max Depth name: flattening_max_depth +- description: '' + kind: object + label: Format Format Csv + name: format.format_csv +- description: '' + kind: object + label: Format Format Json + name: format.format_json +- description: Set true if you want to declare schema of the resulting parquet file + based on taps. Doesn't work with 'anyOf' types or when complex data is not defined + at element level. Doesn't work with validate option for now. + kind: boolean + label: Format Format Parquet Get Schema From Tap + name: format.format_parquet.get_schema_from_tap + value: false - description: This flag determines whether the data types of incoming data elements should be validated. When set `True`, a schema is created from the first record and all subsequent records that don't match that data type are cast. Default False. @@ -139,6 +434,21 @@ settings: label: Include Process Date name: include_process_date value: false +- description: The method to use when loading data into the destination. `append-only` + will always write all input records whether that records already exists or not. + `upsert` will update existing records and insert new records. `overwrite` will + delete all existing records and insert all input records. + kind: options + label: Load Method + name: load_method + options: + - label: Append Only + value: append-only + - label: Upsert + value: upsert + - label: Overwrite + value: overwrite + value: append-only - description: A flag (only works if append_date_to_prefix is enabled) to have partitioning name formatted e.g. 'year=2023/month=01/day=01'. kind: boolean diff --git a/_data/meltano/loaders/target-s3/jkausti.yml b/_data/meltano/loaders/target-s3/jkausti.yml new file mode 100644 index 000000000..509466ed3 --- /dev/null +++ b/_data/meltano/loaders/target-s3/jkausti.yml @@ -0,0 +1,72 @@ +capabilities: +- about +- schema-flattening +- stream-maps +description: AWS S3 - Apache Parquet File Format +domain_url: https://aws.amazon.com/s3/ +executable: target-s3 +keywords: +- meltano_sdk +label: S3 Parquet +logo_url: /assets/logos/loaders/s3-parquet.png +maintenance_status: active +name: target-s3-parquet +namespace: target_s3_parquet +next_steps: '' +pip_url: git+https://github.com/jkausti/target-s3.git +quality: gold +repo: https://github.com/jkausti/target-s3 +settings: +- description: AWS Access Key ID used to authenticate programmatically to S3. + kind: password + label: AWS Access Key ID + name: AWS_ACCESS_KEY_ID +- description: AWS Secret Access Key used to authenticate programmatically to S3. + kind: password + label: AWS Secret Access Key + name: AWS_SECRET_ACCESS_KEY +- description: Specfies the region in which your S3 bucket exists. + kind: string + label: AWS Region + name: aws_region +- description: Specifies the filetype to store the data as. Only parquet supported + atm. *required* + kind: string + label: Filetype + name: filetype + value: parquet +- description: "'True' to enable schema flattening and automatically expand nested + properties." + kind: boolean + label: Flattening Enabled + name: flattening_enabled +- description: The max depth to flatten schemas. + kind: integer + label: Flattening Max Depth + name: flattening_max_depth +- description: "Determines the folder-structure. Alternatives are: 'simple', 'date_hierarchy' + and 'both'." + kind: string + label: Folder Structure + name: folder_structure + value: simple +- description: The path to where the data should be stored. *required* + kind: string + label: Path + name: path +- description: User-defined config values to be used within map expressions. + kind: object + label: Stream Map Config + name: stream_map_config +- description: Config object for stream maps capability. For more information check + out [Stream Maps](https://sdk.meltano.com/en/latest/stream_maps.html). + kind: object + label: Stream Maps + name: stream_maps +settings_group_validation: +- - filetype + - folder_structure + - path +settings_preamble: '' +usage: '' +variant: jkausti diff --git a/_data/meltano/loaders/target-timeplus/timeplus.yml b/_data/meltano/loaders/target-timeplus/timeplus.yml index f9fb8fcbd..3027d2a36 100644 --- a/_data/meltano/loaders/target-timeplus/timeplus.yml +++ b/_data/meltano/loaders/target-timeplus/timeplus.yml @@ -19,7 +19,7 @@ repo: https://github.com/timeplus-io/target-timeplus settings: - description: Personal API key kind: password - label: API Key + label: APIkey name: apikey - description: Timeplus workspace endpoint kind: string