artem_vysotsky
07/05/2021, 1:45 PMdouwe_maan
07/05/2021, 3:30 PMmeltano invoke tap-foo > output.json
2. Take the tap output and pipe it into the target by itself: cat output.json | meltano invoke target-foo
That will tell you whether the slow part is extracting the data from the source, or loading it into the destinationartem_vysotsky
07/05/2021, 4:41 PMartem_vysotsky
07/05/2021, 4:42 PM"""REST client handling, including eodhistoricaldataStream base class."""
import requests
from pathlib import Path
from typing import Any, Dict, Optional
from singer_sdk.streams import RESTStream
SCHEMAS_DIR = Path(__file__).parent / Path("./schemas")
class eodhistoricaldataStream(RESTStream):
"""eodhistoricaldata stream class."""
url_base = "<https://eodhistoricaldata.com/api>"
def get_next_page_token(
self, response: requests.Response, previous_token: Optional[Any]
) -> Optional[Any]:
return None
def get_url_params(
self, context: Optional[dict], next_page_token: Optional[Any]
) -> Dict[str, Any]:
"""Return a dictionary of values to be used in URL parameterization."""
params: dict = {"api_token": self.config['api_token']}
return params
artem_vysotsky
07/05/2021, 4:43 PMartem_vysotsky
07/05/2021, 4:43 PM"""Stream type classes for tap-eodhistoricaldata."""
from pathlib import Path
from typing import Any, Dict, Optional, Iterator
from tap_eodhistoricaldata.client import eodhistoricaldataStream
SCHEMAS_DIR = Path(__file__).parent / Path("./schemas")
class Fundamentals(eodhistoricaldataStream):
"""Define custom stream."""
name = "fundamentals"
path = "/fundamentals/{Code}"
primary_keys = ["Code"]
selected_by_default = True
replication_key = None
schema_filepath = SCHEMAS_DIR / "fundamentals.json"
@property
def partitions(self) -> Iterator[Dict[str, Any]]:
return map(lambda x: {'Code': x}, self.config['symbols'])
def post_process(self, row: dict, context: Optional[dict] = None) -> dict:
row['Code'] = context['Code']
return row
artem_vysotsky
07/05/2021, 4:43 PMartem_vysotsky
07/05/2021, 4:44 PMartem_vysotsky
07/05/2021, 4:44 PM@pytest.mark.vcr
def test_selected():
tap1 = Tapeodhistoricaldata(config=SAMPLE_CONFIG, parse_env_config=True)
tap1.sync_all()
artem_vysotsky
07/05/2021, 4:44 PMartem_vysotsky
07/05/2021, 4:47 PMartem_vysotsky
07/05/2021, 4:47 PM_catalog:27:is_property_selected
artem_vysotsky
07/05/2021, 4:48 PMkeys_order_dependent:4:make_key
douwe_maan
07/05/2021, 4:49 PMdouwe_maan
07/05/2021, 4:49 PMartem_vysotsky
07/05/2021, 4:51 PMdouwe_maan
07/05/2021, 4:53 PMartem_vysotsky
07/05/2021, 4:53 PMdouwe_maan
07/05/2021, 6:10 PMartem_vysotsky
07/05/2021, 7:02 PM