dinh_d___c_d____ng
06/14/2023, 9:55 AMexample file meltano.yml
version: 1
default_environment: dev
project_id: 1b791a52-034d-4c11-a000-ef0eeed7b573
send_anonymous_usage_stats: false
plugins:
extractors:
- name: tap-magento
pip_url: git+<https://github.com/bavaan-platform/tap-magento.git>
executable: tap-magento
config:
username: TAP_MAGENTO_USERNAME
store_url: TAP_MAGENTO_STORE_URL
select:
- products.*
- categories.*
- products_attribute.*
- name: tap-shopify
pip_url: git+<https://github.com/dinhducduong/tap-shopify.git>
config:
store: TAP_SHOPIFY_STORE
admin_url: TAP_SHOPIFY_ADMIN_URL
access_token: TAP_SHOPIFY_ACCESS_TOKEN
select:
- products.*
- custom_collections.*
loaders:
- name: target-elasticsearch
variant: dtmirizzi
pip_url: target-elasticsearch
config:
host: TARGET_ELASTICSEARCH_HOST
index_format: TARGET_ELASTICSEARCH_INDEX_FORMAT
utilities:
- name: dagster
variant: quantile-development
pip_url: git+<https://github.com/quantile-development/dagster-ext.git> -e ${MELTANO_PROJECT_ROOT}/..
mappers:
- name: meltano-map-transformer
variant: meltano
pip_url: git+<https://github.com/MeltanoLabs/meltano-map-transform.git>
settings:
- name: stream_maps
kind: object
mappings:
- name: mask
config:
stream_maps:
products:
id: id
sku: handle
name: title
status: "'1' if status == 'active' else '0'"
media_gallery_entries: image
__else__: __NULL__
jobs:
- name: sync-data-magento
tasks:
- dagster
- tap-magento target-elasticsearch
- name: sync-data-shopify
tasks:
- dagster
- tap-shopify mask target-elasticsearch
schedules:
- name: schedules-tap-magento-target-elasticsearch
interval: "@daily"
job: sync-data-magento
- name: schedules-tap-shopify-target-elasticsearch
interval: "@daily"
job: sync-data-shopify
environments:
- name: dev
- name: staging
- name: prod