Я пытаюсь запустить конвейер локально (Sierra) с помощью Apache Beam, используя предоставленные лучом API-интерфейсы ввода-вывода для Google BigQuery.
Я настроил свою среду с помощью Virtualenv, как было предложено Быстрый старт по Beam Python, и я можно запустить пример wordcount.py. Я также могу правильно запустить настраиваемый конвейер с beam.Create
и beam.ParDo
.
Но я не могу запустить конвейер с BigQuery I / O. Есть идеи, что я делаю неправильно?
Скрипт python следующий.
import apache_beam as beam
from apache_beam.utils.pipeline_options import PipelineOptions
from apache_beam.io import WriteToText
class MyDoFn(beam.DoFn):
def process(self, element):
return element
def run():
opts = {
'project': 'gc-project-name'
}
p = beam.Pipeline(options=PipelineOptions(**opts))
input_query = "SELECT name FROM `gc-project-name.dataset_name.table_name`"
(p
| beam.io.Read(beam.io.BigQuerySource(query=input_query))
| beam.ParDo(MyDoFn())
| beam.io.WriteToText('output.txt')
)
result = p.run()
result.wait_until_finish()
if __name__ == '__main__':
run()
Когда я запускаю его, я получаю следующую ошибку.
WARNING:root:Task failed: Traceback (most recent call last):
File "/Users/localuser/Virtualenvs/abeam/lib/python2.7/site-packages/apache_beam/runners/direct/executor.py", line 300, in __call__
result = evaluator.finish_bundle()
File "/Users/localuser/Virtualenvs/abeam/lib/python2.7/site-packages/apache_beam/runners/direct/transform_evaluator.py", line 208, in finish_bundle
with self._source.reader() as reader:
File "/Users/localuser/Virtualenvs/abeam/lib/python2.7/site-packages/apache_beam/io/gcp/bigquery.py", line 590, in __enter__
self.client = BigQueryWrapper(client=self.test_bigquery_client)
File "/Users/localuser/Virtualenvs/abeam/lib/python2.7/site-packages/apache_beam/io/gcp/bigquery.py", line 682, in __init__
self.client = client or bigquery.BigqueryV2(
AttributeError: 'module' object has no attribute 'BigqueryV2'
Traceback (most recent call last):
File "/Users/localuser/Virtualenvs/abeam/lib/python2.7/site-packages/apache_beam/runners/direct/executor.py", line 300, in __call__
result = evaluator.finish_bundle()
File "/Users/localuser/Virtualenvs/abeam/lib/python2.7/site-packages/apache_beam/runners/direct/transform_evaluator.py", line 208, in finish_bundle
with self._source.reader() as reader:
File "/Users/localuser/Virtualenvs/abeam/lib/python2.7/site-packages/apache_beam/io/gcp/bigquery.py", line 590, in __enter__
self.client = BigQueryWrapper(client=self.test_bigquery_client)
File "/Users/localuser/Virtualenvs/abeam/lib/python2.7/site-packages/apache_beam/io/gcp/bigquery.py", line 682, in __init__
self.client = client or bigquery.BigqueryV2(
AttributeError: 'module' object has no attribute 'BigqueryV2'
WARNING:root:A task failed with exception.
'module' object has no attribute 'BigqueryV2'
Traceback (most recent call last):
File "frombigquery.py", line 54, in <module>
run()
File "frombigquery.py", line 51, in run
result.wait_until_finish()
File "/Users/localuser/Virtualenvs/abeam/lib/python2.7/site-packages/apache_beam/runners/direct/direct_runner.py", line 157, in wait_until_finish
self._executor.await_completion()
File "/Users/localuser/Virtualenvs/abeam/lib/python2.7/site-packages/apache_beam/runners/direct/executor.py", line 335, in await_completion
self._executor.await_completion()
File "/Users/localuser/Virtualenvs/abeam/lib/python2.7/site-packages/apache_beam/runners/direct/executor.py", line 300, in __call__
result = evaluator.finish_bundle()
File "/Users/localuser/Virtualenvs/abeam/lib/python2.7/site-packages/apache_beam/runners/direct/transform_evaluator.py", line 208, in finish_bundle
with self._source.reader() as reader:
File "/Users/localuser/Virtualenvs/abeam/lib/python2.7/site-packages/apache_beam/io/gcp/bigquery.py", line 590, in __enter__
self.client = BigQueryWrapper(client=self.test_bigquery_client)
File "/Users/localuser/Virtualenvs/abeam/lib/python2.7/site-packages/apache_beam/io/gcp/bigquery.py", line 682, in __init__
self.client = client or bigquery.BigqueryV2(
AttributeError: 'module' object has no attribute 'BigqueryV2'
pip install google-apitools
решил проблему. - person MRvaino   schedule 06.05.2017