visch
02/08/2023, 5:40 PMvisch
02/08/2023, 5:42 PMimport os
from datetime import datetime
from distutils.dir_util import copy_tree, remove_tree
import click
from cookiecutter.main import cookiecutter
@click.command()
@click.argument('model_folder')
@click.argument('schema_name')
@click.argument('model_name')
@click.option('--extract_model_name')
@click.option('--extract_source_relation')
@click.option('--cookiecutter_folder', default='cookiecutter/')
@click.option('--cookiecutter_folder', default='cookiecutter/')
@click.option('--output_folder', default='out/', help='Folder I will copy cookiecutter generated files to')
#@click.option('--temp_folder', default='/temp', help='Folder I will generate cookiecutters in')
def generate_models(model_folder, schema_name, model_name, cookiecutter_folder, output_folder, extract_model_name, extract_source_relation):
click.echo(locals())
model_name = model_name.upper()
primary_keys = {}
with open("powerschool_primary_keys.txt", "r") as f:
# Read in each line split by tabs to a dictionary
primary_keys = {line.split("\t")[1]: line.split("\t")[0] for line in f.read().splitlines()}
extra = {
"model_folder":model_folder,
"schema_name":schema_name,
"model_name": model_name,
"output_folder":output_folder,
"extract_model_name": extract_model_name,
"extract_source_relation": extract_source_relation,
"primary_key": primary_keys[model_name],
}
#Generate cookie cutter files
cookiecutter_caller(cookiecutter_folder, extra)
#Move them to output folder
copy_tree(model_folder, output_folder+model_folder+"//")
#Remove old cookie cutter files
remove_tree(model_folder)
#Move to DBT folder
#datamart
source_folder = output_folder+model_folder+"/"+schema_name
db_name = "DataMart"
source_file_name = f"/{db_name}__PowerSchool__"+model_name+".sql"
output_folder = f"../meltano/transform/models/{db_name.lower()}/"+schema_name
os.rename(source_folder + source_file_name, output_folder+source_file_name)
#stage
db_name = "Stage"
source_file_name = f"/{db_name}__PowerSchool__"+model_name+".sql"
output_folder = f"../meltano/transform/models/{db_name.lower()}/"+schema_name
os.rename(source_folder + source_file_name, output_folder+source_file_name)
#update sources.yml
with open(f"../meltano/transform/models/stage/{schema_name}/sources.yml", "a") as sources_file:
sources_file.write("\n - name: PS_"+model_name)
#TODO Meltano.yml
print(f"Run this: python autoidm.py --tap_name=tap-powerschool --target_name=target-mssql --select_filter=\"PS-{model_name}\" --dbt_modelfilter=\"+DataMart__PowerSchool__{model_name}\"")
def cookiecutter_caller(cookiecutter_folder, extra):
cookiecutter(
cookiecutter_folder,
extra_context=extra,
no_input=True
)
if __name__ == '__main__':
generate_models()
Example cookie cutter file
with source as (
{{ '{{' }} cleaned_source('{{cookiecutter.schema_name}}', 'PS_{{cookiecutter.model_name|upper}}') {{ '}}'}}
), final AS (
select
*
from source
)
select * from final
visch
02/08/2023, 5:46 PMvisch
02/08/2023, 5:47 PMvisch
02/08/2023, 5:48 PMthomas_briggs
02/08/2023, 6:01 PMthomas_briggs
02/08/2023, 6:01 PMvisch
02/08/2023, 6:01 PMthomas_briggs
02/08/2023, 7:13 PMmeltano select tap <new_table>.*
and I could use it to run meltano state remove <streamId>
... at least once the state command has a remove option, anyway. ๐ You might also need a meltano utility to raise a PR to get the new/modified files checked in though.thomas_briggs
02/08/2023, 7:14 PM