jan_maly
08/23/2021, 3:13 PM# This configuration is inspired by: <https://pythonspeed.com/articles/activate-conda-dockerfile/>
# use official Docker image for CONDA
FROM continuumio/miniconda3:4.9.2-alpine
WORKDIR /project
# install supervisor, git and bash
RUN apk --update add supervisor git bash
# create the Python environment:
COPY ./requirements.txt .
COPY ./environment.yml .
RUN conda env create -f environment.yml
# make RUN commands use the new environment:
ENV PATH /opt/conda/envs/meltano-demo/bin:$PATH
# ENV variables
# don't allow changes to containerized project files
ENV MELTANO_PROJECT_READONLY true
ENV MELTANO_READONLY true
ENV MELTANO_UI_READONLY true
ENV MELTANO_SEND_ANONYMOUS_USAGE_STATS false
ENV MELTANO_DISABLE_TRACKING true
ENV MELTANO_DISCOVERY_URL false
ENV MELTANO_UI_ANALYSIS false
# install all plugins into the `.meltano` directory
COPY ./meltano.yml .
RUN meltano install
# configuration for supervisord - to run all services
COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
# copy over remaining project files
COPY . .
## install scheduler
#RUN meltano add files airflow
# create ouptut directory - project relevant
RUN mkdir outputs
# expose default port for the app
EXPOSE $PORT
# run services to execute the data pipeline from supervisors
CMD ["/usr/bin/supervisord"]
supervisord.conf:
# Run services to support the Data Pipeline
[supervisord] # this is the main process for the Supervisor
nodaemon=true # this setting is to specify that we are not running in daemon mode
[program:scheduler] # the first service - scheduler
command=/bin/bash -c "meltano invoke airflow scheduler" # this is the main command to run the 1st service
startsecs=0
autostart=true
autorestart=true # this setting specifies that the supervisor will restart the service in case of failure
stderr_logfile=/dev/stdout # this setting specifies that the supervisor will log the errors in the standard output
stderr_logfile_maxbytes = 0
stdout_logfile=/dev/stdout # this setting specifies that the supervisor will log the output in the standard output
stdout_logfile_maxbytes = 0
[program:meltano_ui] # the second service - UI for the Meltano Instance... configuration is similar
command=/bin/bash -c "meltano ui"
autostart=true
autorestart=true
startsecs=0
stderr_logfile=/dev/stdout
stderr_logfile_maxbytes = 0
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes = 0