https://linen.dev logo
Join Slack
Powered by
Has anyone had any experience with<https://hub.mel...
# troubleshooting
h
Has anyone had any experience with tap-bigquery? Having trouble with the 
catalog.json
it generates – seems like the format is not correct
d
What's the error/issue you are seeing?
h
Copy code
Traceback (most recent call last):
  File "/Users/hassansyyid/env/singer-discover/lib/python3.7/site-packages/prompt_toolkit/cache.py", line 34, in get
    return self._data[key]
KeyError: 1

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/Users/hassansyyid/env/singer-discover/bin/singer-discover", line 11, in <module>
    load_entry_point('singer-discover-catalog==0.1.0', 'console_scripts', 'singer-discover')()
  File "/Users/hassansyyid/env/singer-discover/lib/python3.7/site-packages/singer_discover/__init__.py", line 100, in main
    selections = prompt(stream_options)
  File "/Users/hassansyyid/env/singer-discover/lib/python3.7/site-packages/PyInquirer/prompt.py", line 75, in prompt
    eventloop=eventloop)
  File "/Users/hassansyyid/env/singer-discover/lib/python3.7/site-packages/prompt_toolkit/shortcuts.py", line 625, in run_application
    result = cli.run()
  File "/Users/hassansyyid/env/singer-discover/lib/python3.7/site-packages/prompt_toolkit/interface.py", line 413, in run
    self._redraw()
  File "/Users/hassansyyid/env/singer-discover/lib/python3.7/site-packages/prompt_toolkit/interface.py", line 358, in _redraw
    self.renderer.render(self, self.layout, is_done=self.is_done)
  File "/Users/hassansyyid/env/singer-discover/lib/python3.7/site-packages/prompt_toolkit/renderer.py", line 429, in render
    extended_height=size.rows,
  File "/Users/hassansyyid/env/singer-discover/lib/python3.7/site-packages/prompt_toolkit/layout/containers.py", line 142, in write_to_screen
    sizes = self._divide_heigths(cli, write_position)
  File "/Users/hassansyyid/env/singer-discover/lib/python3.7/site-packages/prompt_toolkit/layout/containers.py", line 177, in _divide_heigths
    dimensions = [get_dimension_for_child(c, index) for index, c in enumerate(self.children)]
  File "/Users/hassansyyid/env/singer-discover/lib/python3.7/site-packages/prompt_toolkit/layout/containers.py", line 177, in <listcomp>
    dimensions = [get_dimension_for_child(c, index) for index, c in enumerate(self.children)]
  File "/Users/hassansyyid/env/singer-discover/lib/python3.7/site-packages/prompt_toolkit/layout/containers.py", line 175, in get_dimension_for_child
    return c.preferred_height(cli, write_position.width, write_position.extended_height)
  File "/Users/hassansyyid/env/singer-discover/lib/python3.7/site-packages/prompt_toolkit/layout/containers.py", line 1652, in preferred_height
    return self.content.preferred_height(cli, width, max_available_height)
  File "/Users/hassansyyid/env/singer-discover/lib/python3.7/site-packages/prompt_toolkit/layout/containers.py", line 1000, in preferred_height
    cli, width - total_margin_width, max_available_height, wrap_lines),
  File "/Users/hassansyyid/env/singer-discover/lib/python3.7/site-packages/prompt_toolkit/layout/controls.py", line 254, in preferred_height
    content = self.create_content(cli, width, None)
  File "/Users/hassansyyid/env/singer-discover/lib/python3.7/site-packages/prompt_toolkit/layout/controls.py", line 259, in create_content
    tokens_with_mouse_handlers = self._get_tokens_cached(cli)
  File "/Users/hassansyyid/env/singer-discover/lib/python3.7/site-packages/prompt_toolkit/layout/controls.py", line 239, in _get_tokens_cached
    cli.render_counter, lambda: self.get_tokens(cli))
  File "/Users/hassansyyid/env/singer-discover/lib/python3.7/site-packages/prompt_toolkit/cache.py", line 37, in get
    value = getter_func()
  File "/Users/hassansyyid/env/singer-discover/lib/python3.7/site-packages/prompt_toolkit/layout/controls.py", line 239, in <lambda>
    cli.render_counter, lambda: self.get_tokens(cli))
  File "/Users/hassansyyid/env/singer-discover/lib/python3.7/site-packages/PyInquirer/prompts/checkbox.py", line 99, in _get_choice_tokens
    tokens.pop()  # Remove last newline.
IndexError: pop from empty list
Looks like it’s because it’s not really adding the columns in the table as fields in the catalog
So the table itself shows as a stream, but the stream has no fields in the catalog that you can select
d
Running this config:
Copy code
{
  "streams": [
    {
      "name": "baseball_schedule",
      "table": "`bigquery-public-data.baseball.schedules`",
      "columns": "*",
      "datetime_key": "created"
    }
  ],
  "start_datetime": "2015-01-01T00:00:00Z"
}
seems to be ok when just running discovery on the tap.
Copy code
{
  "streams": [
    {
      "stream": "baseball_schedule",
      "tap_stream_id": "baseball_schedule",
      "schema": {
        "selected": true,
        "type": "object",
        "stream": "baseball_schedule",
        "key_properties": [],
        "properties": {
          "gameId": {
            "type": [
              "null",
              "string"
            ]
          },
          "gameNumber": {
            "type": [
              "null",
              "integer"
            ]
          },
          "seasonId": {
            "type": [
              "null",
              "string"
            ]
          },
          "year": {
            "type": [
              "null",
              "integer"
            ]
          },
          "type": {
            "type": [
              "null",
              "string"
            ]
          },
          "dayNight": {
            "type": [
              "null",
              "string"
            ]
          },
          "duration": {
            "type": [
              "null",
              "string"
            ]
          },
          "duration_minutes": {
            "type": [
              "null",
              "integer"
            ]
          },
          "homeTeamId": {
            "type": [
              "null",
              "string"
            ]
          },
          "homeTeamName": {
            "type": [
              "null",
              "string"
            ]
          },
          "awayTeamId": {
            "type": [
              "null",
              "string"
            ]
          },
          "awayTeamName": {
            "type": [
              "null",
              "string"
            ]
          },
          "startTime": {
            "type": [
              "null",
              "string"
            ],
            "format": "date-time"
          },
          "attendance": {
            "type": [
              "null",
              "integer"
            ]
          },
          "status": {
            "type": [
              "null",
              "string"
            ]
          },
          "created": {
            "type": [
              "null",
              "string"
            ],
            "format": "date-time"
          },
          "_sdc_extracted_at": {
            "type": [
              "null",
              "string"
            ],
            "format": "date-time"
          },
          "_sdc_batched_at": {
            "type": [
              "null",
              "string"
            ],
            "format": "date-time"
          },
          "_etl_tstamp": {
            "type": [
              "null",
              "number"
            ],
            "inclusion": "automatic"
          }
        }
      },
      "metadata": [
        {
          "metadata": {
            "selected": true,
            "table": "`bigquery-public-data.baseball.schedules`",
            "columns": "*",
            "filters": [],
            "datetime_key": "created"
          },
          "breadcrumb": []
        }
      ],
      "key_properties": []
    }
  ]
}
h
@dan_ladd Not an error in the tap, the error is when you use the 
catalog.json
it generates with the singer-discover utility
2 Views
Open in Slack
PreviousNext
Powered by