meltano.yml

default_environment: dev
project_id: eacd5faa-7c52-4e9a-99e5-f08a6843097b
environments:
- name: dev
  env:
    MELTANO_DATABASE_URI: <postgresql://watermelon:watermelon123@ec2-54-255-38-151.ap-southeast-1.compute.amazonaws.com:30003/wmebservices>
- name: staging
  env:
    MELTANO_DATABASE_URI: <postgresql://meltano:meltano@172.16.24.22:5432/meltano_stag>
- name: prod
  env:
    MELTANO_DATABASE_URI: <postgresql://meltano:meltano@172.16.24.22:5432/meltano_prod>
plugins:
  extractors:
  - name: tap-splunk_gw_1
    namespace: tap_splunk
    pip_url: -e ./tap-splunk_gw
    executable: tap-splunk_gw
    capabilities:
    - state
    - catalog
    - discover
    config:
      api_token:
      api_url: 
      clientId: 
      clientSecret: 
      search_query: 
      initial_start_time: 1775327400000
      #endtimeU: 1771266600000
      stream_maps:
        SplunkQuery_RestApi:
          domain_id: config['domain_id']
          tool_name: config['tool_name']
          app_name: config['app_name']
          domain_name: config['domain_name']
          map_id: config['map_id']
          datasource_type: config['datasource_type']
          data_type: config['data_type']
          index_id: config['index_id']
          client_name: config['client_name']
          app_id: config['app_id']
          client_id: config['client_id']
      stream_map_config:
        domain_id: 53
        tool_name: SPLUNK
        app_name: SplunkTest
        domain_name: Retail
        map_id: 11102
        datasource_type: LOGS
        data_type: APPLICATION
        index_id: watermelon_splunktest_10902
        client_name: watermelon
        app_id: 10902
        client_id: 1
  loaders:
  - name: loader-kafka_1
    namespace: kafka_loader
    pip_url: -e ./loader-kafka
    executable: loader-kafka
    config:
      kafka_brokers: ************************
      topic_prefix: splunk
  - name: loader-kafka_2
    namespace: kafka_loader
    pip_url: -e ./loader-kafka
    executable: loader-kafka
    config:
      kafka_brokers: ****************************
      topic_prefix: splunk_2
  files:
  - name: files-airflow
    variant: meltano
    pip_url: git+<https://github.com/meltano/files-airflow.git>
  utilities:
  - name: airflow
    namespace: airflow
    pip_url: >
      git+<https://github.com/meltano/edk.git@main>
      git+<https://github.com/monomeric/airflow-ext.git@main>
      apache-airflow==3.2.0
      --constraint <https://raw.githubusercontent.com/apache/airflow/constraints-3.2.0/constraints-no-providers-${MELTANO__PYTHON_VERSION}.txt>
    executable: airflow_invoker
    settings:
    - name: database.sql_alchemy_conn
      env: AIRFLOW__DATABASE__SQL_ALCHEMY_CONN
      value: sqlite:///$MELTANO_PROJECT_ROOT/.meltano/utilities/airflow/airflow.db
      label: SQL Alchemy Connection
    - name: core.dags_folder
      env: AIRFLOW__CORE__DAGS_FOLDER
      value: $MELTANO_PROJECT_ROOT/orchestrate/airflow/dags
      label: DAGs Folder
    - name: core.plugins_folder
      env: AIRFLOW__CORE__PLUGINS_FOLDER
      value: $MELTANO_PROJECT_ROOT/orchestrate/airflow/plugins
      label: Plugins Folder
    - name: core.load_examples
      env: AIRFLOW__CORE__LOAD_EXAMPLES
      value: false
      label: Load Examples
    - name: core.dags_are_paused_at_creation
      env: AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION
      value: false
      label: Pause DAGs at Creation
    - name: webserver.web_server_port
      env: AIRFLOW__API__PORT
      value: 8080
      label: Webserver Port
      description: |
        The web and API server port, formerly set by AIRFLOW__WEBSERVER__WEB_SERVER_PORT in Airflow 2.x. If this variable
        remains unset with Airflow 3.x, configuration generation may fail as the airflow.cfg file catches debug logs through
        'stdout' upon creation by 'airflow config list --defaults'.
    - name: logging.base_log_folder
      env: AIRFLOW__LOGGING__BASE_LOG_FOLDER
      value: $MELTANO_PROJECT_ROOT/.meltano/utilities/airflow/logs
      label: Base Log Folder
      description: |
        The folder where airflow should store its log files. This path must be absolute. There are a few existing
        configurations that assume this is set to the default. If you choose to override this you may need to update
        the dag_processor_manager_log_location and child_process_log_directory settings as well.
    - name: logging.dag_processor_manager_log_location
      env: AIRFLOW__LOGGING__DAG_PROCESSOR_MANAGER_LOG_LOCATION
      value: $MELTANO_PROJECT_ROOT/.meltano/utilities/airflow/logs/dag_processor_manager/dag_processor_manager.log
      label: Dag Processor Manager Log Location
      description: |
        Where to send dag parser logs.
    - name: scheduler.child_process_log_directory
      env: AIRFLOW__SCHEDULER__CHILD_PROCESS_LOG_DIRECTORY
      value: $MELTANO_PROJECT_ROOT/.meltano/utilities/airflow/logs/scheduler
      label: Child Process Log Directory
      description: |
        Where to send the logs of each scheduler process.
    - name: extension.airflow_home
      env: AIRFLOW_HOME
      value: $MELTANO_PROJECT_ROOT/orchestrate/airflow
      label: Airflow Home
      description: |
        The directory where Airflow will store its configuration, logs, and other files.
    - name: extension.airflow_config
      env: AIRFLOW_CONFIG
      value: $MELTANO_PROJECT_ROOT/orchestrate/airflow/airflow.cfg
      label: Airflow Home
      description: |
        The path where the Airflow configuration file will be stored.
    commands:
      describe:
        args: describe
        executable: airflow_extension
      initialize:
        args: initialize
        executable: airflow_extension
      invoke:
        args: invoke
        executable: airflow_extension
schedules:
- name: cw_1
  interval: '*/5 * * * *'
  extractor: tap-splunk_gw_1
  loader: loader-kafka_1
  transform: skip
  start_date: 2023-07-01 00:00:00