2019-11-05 21:42:16.574 PSTError message from worker: java.util.concurrent.ExecutionException: java.lang.RuntimeException: Error received from SDK harness for instruction -450911: Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/sdk_worker.py", line 151, in _execute response = task() File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/sdk_worker.py", line 186, in <lambda> self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/sdk_worker.py", line 265, in do_instruction request.instruction_id) File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/sdk_worker.py", line 281, in process_bundle delayed_applications = bundle_processor.process_bundle(instruction_id) File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/bundle_processor.py", line 547, in process_bundle ].process_encoded(data.data) File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/bundle_processor.py", line 139, in process_encoded self.output(decoded_value) File "apache_beam/runners/worker/operations.py", line 222, in apache_beam.runners.worker.operations.Operation.output def output(self, windowed_value, output_index=0): File "apache_beam/runners/worker/operations.py", line 223, in apache_beam.runners.worker.operations.Operation.output cython.cast(Receiver, self.receivers[output_index]).receive(windowed_value) File "apache_beam/runners/worker/operations.py", line 131, in apache_beam.runners.worker.operations.SingletonConsumerSet.receive self.consumer.process(windowed_value) File "apache_beam/runners/worker/operations.py", line 537, in apache_beam.runners.worker.operations.DoOperation.process with self.scoped_process_state: File "apache_beam/runners/worker/operations.py", line 538, in apache_beam.runners.worker.operations.DoOperation.process delayed_application = self.dofn_receiver.receive(o) File "apache_beam/runners/common.py", line 723, in apache_beam.runners.common.DoFnRunner.receive self.process(windowed_value) File "apache_beam/runners/common.py", line 729, in apache_beam.runners.common.DoFnRunner.process self._reraise_augmented(exn) File "apache_beam/runners/common.py", line 762, in apache_beam.runners.common.DoFnRunner._reraise_augmented raise File "apache_beam/runners/common.py", line 727, in apache_beam.runners.common.DoFnRunner.process return self.do_fn_invoker.invoke_process(windowed_value) File "apache_beam/runners/common.py", line 555, in apache_beam.runners.common.PerWindowInvoker.invoke_process self._invoke_per_window( File "apache_beam/runners/common.py", line 624, in apache_beam.runners.common.PerWindowInvoker._invoke_per_window output_processor.process_outputs( File "apache_beam/runners/common.py", line 808, in apache_beam.runners.common._OutputProcessor.process_outputs def process_outputs(self, windowed_input_element, results): File "apache_beam/runners/common.py", line 847, in apache_beam.runners.common._OutputProcessor.process_outputs self.main_receivers.receive(windowed_value) File "apache_beam/runners/worker/operations.py", line 131, in apache_beam.runners.worker.operations.SingletonConsumerSet.receive self.consumer.process(windowed_value) File "apache_beam/runners/worker/operations.py", line 537, in apache_beam.runners.worker.operations.DoOperation.process with self.scoped_process_state: File "apache_beam/runners/worker/operations.py", line 538, in apache_beam.runners.worker.operations.DoOperation.process delayed_application = self.dofn_receiver.receive(o) File "apache_beam/runners/common.py", line 723, in apache_beam.runners.common.DoFnRunner.receive self.process(windowed_value) File "apache_beam/runners/common.py", line 729, in apache_beam.runners.common.DoFnRunner.process self._reraise_augmented(exn) File "apache_beam/runners/common.py", line 762, in apache_beam.runners.common.DoFnRunner._reraise_augmented raise File "apache_beam/runners/common.py", line 727, in apache_beam.runners.common.DoFnRunner.process return self.do_fn_invoker.invoke_process(windowed_value) File "apache_beam/runners/common.py", line 555, in apache_beam.runners.common.PerWindowInvoker.invoke_process self._invoke_per_window( File "apache_beam/runners/common.py", line 624, in apache_beam.runners.common.PerWindowInvoker._invoke_per_window output_processor.process_outputs( File "apache_beam/runners/common.py", line 808, in apache_beam.runners.common._OutputProcessor.process_outputs def process_outputs(self, windowed_input_element, results): File "apache_beam/runners/common.py", line 847, in apache_beam.runners.common._OutputProcessor.process_outputs self.main_receivers.receive(windowed_value) File "apache_beam/runners/worker/operations.py", line 131, in apache_beam.runners.worker.operations.SingletonConsumerSet.receive self.consumer.process(windowed_value) File "apache_beam/runners/worker/operations.py", line 537, in apache_beam.runners.worker.operations.DoOperation.process with self.scoped_process_state: File "apache_beam/runners/worker/operations.py", line 538, in apache_beam.runners.worker.operations.DoOperation.process delayed_application = self.dofn_receiver.receive(o) File "apache_beam/runners/common.py", line 723, in apache_beam.runners.common.DoFnRunner.receive self.process(windowed_value) File "apache_beam/runners/common.py", line 729, in apache_beam.runners.common.DoFnRunner.process self._reraise_augmented(exn) File "apache_beam/runners/common.py", line 777, in apache_beam.runners.common.DoFnRunner._reraise_augmented raise_with_traceback(new_exn) File "apache_beam/runners/common.py", line 727, in apache_beam.runners.common.DoFnRunner.process return self.do_fn_invoker.invoke_process(windowed_value) File "apache_beam/runners/common.py", line 419, in apache_beam.runners.common.SimpleInvoker.invoke_process windowed_value, self.process_method(windowed_value.value)) File "streaming_twitter.py", line 188, in <lambda> File "streaming_twitter.py", line 102, in estimate File "streaming_twitter.py", line 84, in estimate_aiplatform File "streaming_twitter.py", line 42, in get_service File "/usr/local/lib/python2.7/dist-packages/googleapiclient/_helpers.py", line 130, in positional_wrapper return wrapped(*args, **kwargs) File "/usr/local/lib/python2.7/dist-packages/googleapiclient/discovery.py", line 227, in build credentials=credentials) File "/usr/local/lib/python2.7/dist-packages/googleapiclient/_helpers.py", line 130, in positional_wrapper return wrapped(*args, **kwargs) File "/usr/local/lib/python2.7/dist-packages/googleapiclient/discovery.py", line 363, in build_from_document credentials = _auth.default_credentials() File "/usr/local/lib/python2.7/dist-packages/googleapiclient/_auth.py", line 42, in default_credentials credentials, _ = google.auth.default() File "/usr/local/lib/python2.7/dist-packages/google/auth/_default.py", line 306, in default raise exceptions.DefaultCredentialsError(_HELP_MESSAGE) DefaultCredentialsError: Could not automatically determine credentials. Please set GOOGLE_APPLICATION_CREDENTIALS or explicitly create credentials and re-run the application. For more information, please see https://cloud.google.com/docs/authentication/getting-started [while running 'generatedPtransform-450901'] java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:357) java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1895) org.apache.beam.sdk.util.MoreFutures.get(MoreFutures.java:57) org.apache.beam.runners.dataflow.worker.fn.control.RegisterAndProcessBundleOperation.finish(RegisterAndProcessBundleOperation.java:281) org.apache.beam.runners.dataflow.worker.util.common.worker.MapTaskExecutor.execute(MapTaskExecutor.java:85) org.apache.beam.runners.dataflow.worker.fn.control.BeamFnMapTaskExecutor.execute(BeamFnMapTaskExecutor.java:125) org.apache.beam.runners.dataflow.worker.StreamingDataflowWorker.process(StreamingDataflowWorker.java:1269) org.apache.beam.runners.dataflow.worker.StreamingDataflowWorker.access$1000(StreamingDataflowWorker.java:146) org.apache.beam.runners.dataflow.worker.StreamingDataflowWorker$6.run(StreamingDataflowWorker.java:1008) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) java.lang.Thread.run(Thread.java:745) Caused by: java.lang.RuntimeException: Error received from SDK harness for instruction -450911: Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/sdk_worker.py", line 151, in _execute response = task() File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/sdk_worker.py", line 186, in <lambda> self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/sdk_worker.py", line 265, in do_instruction request.instruction_id) File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/sdk_worker.py", line 281, in process_bundle delayed_applications = bundle_processor.process_bundle(instruction_id) File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/bundle_processor.py", line 547, in process_bundle ].process_encoded(data.data) File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/bundle_processor.py", line 139, in process_encoded self.output(decoded_value) File "apache_beam/runners/worker/operations.py", line 222, in apache_beam.runners.worker.operations.Operation.output def output(self, windowed_value, output_index=0): File "apache_beam/runners/worker/operations.py", line 223, in apache_beam.runners.worker.operations.Operation.output cython.cast(Receiver, self.receivers[output_index]).receive(windowed_value) File "apache_beam/runners/worker/operations.py", line 131, in apache_beam.runners.worker.operations.SingletonConsumerSet.receive self.consumer.process(windowed_value) File "apache_beam/runners/worker/operations.py", line 537, in apache_beam.runners.worker.operations.DoOperation.process with self.scoped_process_state: File "apache_beam/runners/worker/operations.py", line 538, in apache_beam.runners.worker.operations.DoOperation.process delayed_application = self.dofn_receiver.receive(o) File "apache_beam/runners/common.py", line 723, in apache_beam.runners.common.DoFnRunner.receive self.process(windowed_value) File "apache_beam/runners/common.py", line 729, in apache_beam.runners.common.DoFnRunner.process self._reraise_augmented(exn) File "apache_beam/runners/common.py", line 762, in apache_beam.runners.common.DoFnRunner._reraise_augmented raise File "apache_beam/runners/common.py", line 727, in apache_beam.runners.common.DoFnRunner.process return self.do_fn_invoker.invoke_process(windowed_value) File "apache_beam/runners/common.py", line 555, in apache_beam.runners.common.PerWindowInvoker.invoke_process self._invoke_per_window( File "apache_beam/runners/common.py", line 624, in apache_beam.runners.common.PerWindowInvoker._invoke_per_window output_processor.process_outputs( File "apache_beam/runners/common.py", line 808, in apache_beam.runners.common._OutputProcessor.process_outputs def process_outputs(self, windowed_input_element, results): File "apache_beam/runners/common.py", line 847, in apache_beam.runners.common._OutputProcessor.process_outputs self.main_receivers.receive(windowed_value) File "apache_beam/runners/worker/operations.py", line 131, in apache_beam.runners.worker.operations.SingletonConsumerSet.receive self.consumer.process(windowed_value) File "apache_beam/runners/worker/operations.py", line 537, in apache_beam.runners.worker.operations.DoOperation.process with self.scoped_process_state: File "apache_beam/runners/worker/operations.py", line 538, in apache_beam.runners.worker.operations.DoOperation.process delayed_application = self.dofn_receiver.receive(o) File "apache_beam/runners/common.py", line 723, in apache_beam.runners.common.DoFnRunner.receive self.process(windowed_value) File "apache_beam/runners/common.py", line 729, in apache_beam.runners.common.DoFnRunner.process self._reraise_augmented(exn) File "apache_beam/runners/common.py", line 762, in apache_beam.runners.common.DoFnRunner._reraise_augmented raise File "apache_beam/runners/common.py", line 727, in apache_beam.runners.common.DoFnRunner.process return self.do_fn_invoker.invoke_process(windowed_value) File "apache_beam/runners/common.py", line 555, in apache_beam.runners.common.PerWindowInvoker.invoke_process self._invoke_per_window( File "apache_beam/runners/common.py", line 624, in apache_beam.runners.common.PerWindowInvoker._invoke_per_window output_processor.process_outputs( File "apache_beam/runners/common.py", line 808, in apache_beam.runners.common._OutputProcessor.process_outputs def process_outputs(self, windowed_input_element, results): File "apache_beam/runners/common.py", line 847, in apache_beam.runners.common._OutputProcessor.process_outputs self.main_receivers.receive(windowed_value) File "apache_beam/runners/worker/operations.py", line 131, in apache_beam.runners.worker.operations.SingletonConsumerSet.receive self.consumer.process(windowed_value) File "apache_beam/runners/worker/operations.py", line 537, in apache_beam.runners.worker.operations.DoOperation.process with self.scoped_process_state: File "apache_beam/runners/worker/operations.py", line 538, in apache_beam.runners.worker.operations.DoOperation.process delayed_application = self.dofn_receiver.receive(o) File "apache_beam/runners/common.py", line 723, in apache_beam.runners.common.DoFnRunner.receive self.process(windowed_value) File "apache_beam/runners/common.py", line 729, in apache_beam.runners.common.DoFnRunner.process self._reraise_augmented(exn) File "apache_beam/runners/common.py", line 777, in apache_beam.runners.common.DoFnRunner._reraise_augmented raise_with_traceback(new_exn) File "apache_beam/runners/common.py", line 727, in apache_beam.runners.common.DoFnRunner.process return self.do_fn_invoker.invoke_process(windowed_value) File "apache_beam/runners/common.py", line 419, in apache_beam.runners.common.SimpleInvoker.invoke_process windowed_value, self.process_method(windowed_value.value)) File "streaming_twitter.py", line 188, in <lambda> File "streaming_twitter.py", line 102, in estimate File "streaming_twitter.py", line 84, in estimate_aiplatform File "streaming_twitter.py", line 42, in get_service File "/usr/local/lib/python2.7/dist-packages/googleapiclient/_helpers.py", line 130, in positional_wrapper return wrapped(*args, **kwargs) File "/usr/local/lib/python2.7/dist-packages/googleapiclient/discovery.py", line 227, in build credentials=credentials) File "/usr/local/lib/python2.7/dist-packages/googleapiclient/_helpers.py", line 130, in positional_wrapper return wrapped(*args, **kwargs) File "/usr/local/lib/python2.7/dist-packages/googleapiclient/discovery.py", line 363, in build_from_document credentials = _auth.default_credentials() File "/usr/local/lib/python2.7/dist-packages/googleapiclient/_auth.py", line 42, in default_credentials credentials, _ = google.auth.default() File "/usr/local/lib/python2.7/dist-packages/google/auth/_default.py", line 306, in default raise exceptions.DefaultCredentialsError(_HELP_MESSAGE) DefaultCredentialsError: Could not automatically determine credentials. Please set GOOGLE_APPLICATION_CREDENTIALS or explicitly create credentials and re-run the application. For more information, please see https://cloud.google.com/docs/authentication/getting-started [while running 'generatedPtransform-450901'] org.apache.beam.runners.fnexecution.control.FnApiControlClient$ResponseStreamObserver.onNext(FnApiControlClient.java:157) org.apache.beam.runners.fnexecution.control.FnApiControlClient$ResponseStreamObserver.onNext(FnApiControlClient.java:140) org.apache.beam.vendor.grpc.v1p13p1.io.grpc.stub.ServerCalls$StreamingServerCallHandler$StreamingServerCallListener.onMessage(ServerCalls.java:248) org.apache.beam.vendor.grpc.v1p13p1.io.grpc.ForwardingServerCallListener.onMessage(ForwardingServerCallListener.java:33) org.apache.beam.vendor.grpc.v1p13p1.io.grpc.Contexts$ContextualizedServerCallListener.onMessage(Contexts.java:76) org.apache.beam.vendor.grpc.v1p13p1.io.grpc.internal.ServerCallImpl$ServerStreamListenerImpl.messagesAvailable(ServerCallImpl.java:263) org.apache.beam.vendor.grpc.v1p13p1.io.grpc.internal.ServerImpl$JumpToApplicationThreadServerStreamListener$1MessagesAvailable.runInContext(ServerImpl.java:683) org.apache.beam.vendor.grpc.v1p13p1.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) org.apache.beam.vendor.grpc.v1p13p1.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) java.lang.Thread.run(Thread.java:745) java.util.concurrent.ExecutionException: java.lang.RuntimeException: Error received from SDK harness for instruction -450803: Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/sdk_worker.py", line 151, in _execute response = task() File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/sdk_worker.py", line 186, in <lambda> self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/sdk_worker.py", line 265, in do_instruction request.instruction_id) File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/sdk_worker.py", line 281, in process_bundle delayed_applications = bundle_processor.process_bundle(instruction_id) File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/bundle_processor.py", line 547, in process_bundle ].process_encoded(data.data) File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/bundle_processor.py", line 139, in process_encoded self.output(decoded_value) File "apache_beam/runners/worker/operations.py", line 222, in apache_beam.runners.worker.operations.Operation.output def output(self, windowed_value, output_index=0): File "apache_beam/runners/worker/operations.py", line 223, in apache_beam.runners.worker.operations.Operation.output cython.cast(Receiver, self.receivers[output_index]).receive(windowed_value) File "apache_beam/runners/worker/operations.py", line 131, in apache_beam.runners.worker.operations.SingletonConsumerSet.receive self.consumer.process(windowed_value) File "apache_beam/runners/worker/operations.py", line 537, in apache_beam.runners.worker.operations.DoOperation.process with self.scoped_process_state: File "apache_beam/runners/worker/operations.py", line 538, in apache_beam.runners.worker.operations.DoOperation.process delayed_application = self.dofn_receiver.receive(o) File "apache_beam/runners/common.py", line 723, in apache_beam.runners.common.DoFnRunner.receive self.process(windowed_value) File "apache_beam/runners/common.py", line 729, in apache_beam.runners.common.DoFnRunner.process self._reraise_augmented(exn) File "apache_beam/runners/common.py", line 762, in apache_beam.runners.common.DoFnRunner._reraise_augmented raise File "apache_beam/runners/common.py", line 727, in apache_beam.runners.common.DoFnRunner.process return self.do_fn_invoker.invoke_process(windowed_value) File "apache_beam/runners/common.py", line 555, in apache_beam.runners.common.PerWindowInvoker.invoke_process self._invoke_per_window( File "apache_beam/runners/common.py", line 624, in apache_beam.runners.common.PerWindowInvoker._invoke_per_window output_processor.process_outputs( File "apache_beam/runners/common.py", line 808, in apache_beam.runners.common._OutputProcessor.process_outputs def process_outputs(self, windowed_input_element, results): File "apache_beam/runners/common.py", line 847, in apache_beam.runners.common._OutputProcessor.process_outputs self.main_receivers.receive(windowed_value) File "apache_beam/runners/worker/operations.py", line 131, in apache_beam.runners.worker.operations.SingletonConsumerSet.receive self.consumer.process(windowed_value) File "apache_beam/runners/worker/operations.py", line 537, in apache_beam.runners.worker.operations.DoOperation.process with self.scoped_process_state: File "apache_beam/runners/worker/operations.py", line 538, in apache_beam.runners.worker.operations.DoOperation.process delayed_application = self.dofn_receiver.receive(o) File "apache_beam/runners/common.py", line 723, in apache_beam.runners.common.DoFnRunner.receive self.process(windowed_value) File "apache_beam/runners/common.py", line 729, in apache_beam.runners.common.DoFnRunner.process self._reraise_augmented(exn) File "apache_beam/runners/common.py", line 762, in apache_beam.runners.common.DoFnRunner._reraise_augmented raise File "apache_beam/runners/common.py", line 727, in apache_beam.runners.common.DoFnRunner.process return self.do_fn_invoker.invoke_process(windowed_value) File "apache_beam/runners/common.py", line 555, in apache_beam.runners.common.PerWindowInvoker.invoke_process self._invoke_per_window( File "apache_beam/runners/common.py", line 624, in apache_beam.runners.common.PerWindowInvoker._invoke_per_window output_processor.process_outputs( File "apache_beam/runners/common.py", line 808, in apache_beam.runners.common._OutputProcessor.process_outputs def process_outputs(self, windowed_input_element, results): File "apache_beam/runners/common.py", line 847, in apache_beam.runners.common._OutputProcessor.process_outputs self.main_receivers.receive(windowed_value) File "apache_beam/runners/worker/operations.py", line 131, in apache_beam.runners.worker.operations.SingletonConsumerSet.receive self.consumer.process(windowed_value) File "apache_beam/runners/worker/operations.py", line 537, in apache_beam.runners.worker.operations.DoOperation.process with self.scoped_process_state: File "apache_beam/runners/worker/operations.py", line 538, in apache_beam.runners.worker.operations.DoOperation.process delayed_application = self.dofn_receiver.receive(o) File "apache_beam/runners/common.py", line 723, in apache_beam.runners.common.DoFnRunner.receive self.process(windowed_value) File "apache_beam/runners/common.py", line 729, in apache_beam.runners.common.DoFnRunner.process self._reraise_augmented(exn) File "apache_beam/runners/common.py", line 777, in apache_beam.runners.common.DoFnRunner._reraise_augmented raise_with_traceback(new_exn) File "apache_beam/runners/common.py", line 727, in apache_beam.runners.common.DoFnRunner.process return self.do_fn_invoker.invoke_process(windowed_value) File "apache_beam/runners/common.py", line 419, in apache_beam.runners.common.SimpleInvoker.invoke_process windowed_value, self.process_method(windowed_value.value)) File "streaming_twitter.py", line 188, in <lambda> File "streaming_twitter.py", line 102, in estimate File "streaming_twitter.py", line 84, in estimate_aiplatform File "streaming_twitter.py", line 42, in get_service File "/usr/local/lib/python2.7/dist-packages/googleapiclient/_helpers.py", line 130, in positional_wrapper return wrapped(*args, **kwargs) File "/usr/local/lib/python2.7/dist-packages/googleapiclient/discovery.py", line 227, in build credentials=credentials) File "/usr/local/lib/python2.7/dist-packages/googleapiclient/_helpers.py", line 130, in positional_wrapper return wrapped(*args, **kwargs) File "/usr/local/lib/python2.7/dist-packages/googleapiclient/discovery.py", line 363, in build_from_document credentials = _auth.default_credentials() File "/usr/local/lib/python2.7/dist-packages/googleapiclient/_auth.py", line 42, in default_credentials credentials, _ = google.auth.default() File "/usr/local/lib/python2.7/dist-packages/google/auth/_default.py", line 306, in default raise exceptions.DefaultCredentialsError(_HELP_MESSAGE) DefaultCredentialsError: Could not automatically determine credentials. Please set GOOGLE_APPLICATION_CREDENTIALS or explicitly create credentials and re-run the application. For more information, please see https://cloud.google.com/docs/authentication/getting-started [while running 'generatedPtransform-450797'] java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:357) java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1895) org.apache.beam.sdk.util.MoreFutures.get(MoreFutures.java:57) org.apache.beam.runners.dataflow.worker.fn.control.RegisterAndProcessBundleOperation.finish(RegisterAndProcessBundleOperation.java:281) org.apache.beam.runners.dataflow.worker.util.common.worker.MapTaskExecutor.execute(MapTaskExecutor.java:85) org.apache.beam.runners.dataflow.worker.fn.control.BeamFnMapTaskExecutor.execute(BeamFnMapTaskExecutor.java:125) org.apache.beam.runners.dataflow.worker.StreamingDataflowWorker.process(StreamingDataflowWorker.java:1269) org.apache.beam.runners.dataflow.worker.StreamingDataflowWorker.access$1000(StreamingDataflowWorker.java:146) org.apache.beam.runners.dataflow.worker.StreamingDataflowWorker$6.run(StreamingDataflowWorker.java:1008) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) java.lang.Thread.run(Thread.java:745) Caused by: java.lang.RuntimeException: Error received from SDK harness for instruction -450803: Traceback (most recent call last): File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/sdk_worker.py", line 151, in _execute response = task() File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/sdk_worker.py", line 186, in <lambda> self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/sdk_worker.py", line 265, in do_instruction request.instruction_id) File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/sdk_worker.py", line 281, in process_bundle delayed_applications = bundle_processor.process_bundle(instruction_id) File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/bundle_processor.py", line 547, in process_bundle ].process_encoded(data.data) File "/usr/local/lib/python2.7/dist-packages/apache_beam/runners/worker/bundle_processor.py", line 139, in process_encoded self.output(decoded_value) File "apache_beam/runners/worker/operations.py", line 222, in apache_beam.runners.worker.operations.Operation.output def output(self, windowed_value, output_index=0): File "apache_beam/runners/worker/operations.py", line 223, in apache_beam.runners.worker.operations.Operation.output cython.cast(Receiver, self.receivers[output_index]).receive(windowed_value) File "apache_beam/runners/worker/operations.py", line 131, in apache_beam.runners.worker.operations.SingletonConsumerSet.receive self.consumer.process(windowed_value) File "apache_beam/runners/worker/operations.py", line 537, in apache_beam.runners.worker.operations.DoOperation.process with self.scoped_process_state: File "apache_beam/runners/worker/operations.py", line 538, in apache_beam.runners.worker.operations.DoOperation.process delayed_application = self.dofn_receiver.receive(o) File "apache_beam/runners/common.py", line 723, in apache_beam.runners.common.DoFnRunner.receive self.process(windowed_value) File "apache_beam/runners/common.py", line 729, in apache_beam.runners.common.DoFnRunner.process self._reraise_augmented(exn) File "apache_beam/runners/common.py", line 762, in apache_beam.runners.common.DoFnRunner._reraise_augmented raise File "apache_beam/runners/common.py", line 727, in apache_beam.runners.common.DoFnRunner.process return self.do_fn_invoker.invoke_process(windowed_value) File "apache_beam/runners/common.py", line 555, in apache_beam.runners.common.PerWindowInvoker.invoke_process self._invoke_per_window( File "apache_beam/runners/common.py", line 624, in apache_beam.runners.common.PerWindowInvoker._invoke_per_window output_processor.process_outputs( File "apache_beam/runners/common.py", line 808, in apache_beam.runners.common._OutputProcessor.process_outputs def process_outputs(self, windowed_input_element, results): File "apache_beam/runners/common.py", line 847, in apache_beam.runners.common._OutputProcessor.process_outputs self.main_receivers.receive(windowed_value) File "apache_beam/runners/worker/operations.py", line 131, in apache_beam.runners.worker.operations.SingletonConsumerSet.receive self.consumer.process(windowed_value) File "apache_beam/runners/worker/operations.py", line 537, in apache_beam.runners.worker.operations.DoOperation.process with self.scoped_process_state: File "apache_beam/runners/worker/operations.py", line 538, in apache_beam.runners.worker.operations.DoOperation.process delayed_application = self.dofn_receiver.receive(o) File "apache_beam/runners/common.py", line 723, in apache_beam.runners.common.DoFnRunner.receive self.process(windowed_value) File "apache_beam/runners/common.py", line 729, in apache_beam.runners.common.DoFnRunner.process self._reraise_augmented(exn) File "apache_beam/runners/common.py", line 762, in apache_beam.runners.common.DoFnRunner._reraise_augmented raise File "apache_beam/runners/common.py", line 727, in apache_beam.runners.common.DoFnRunner.process return self.do_fn_invoker.invoke_process(windowed_value) File "apache_beam/runners/common.py", line 555, in apache_beam.runners.common.PerWindowInvoker.invoke_process self._invoke_per_window( File "apache_beam/runners/common.py", line 624, in apache_beam.runners.common.PerWindowInvoker._invoke_per_window output_processor.process_outputs( File "apache_beam/runners/common.py", line 808, in apache_beam.runners.common._OutputProcessor.process_outputs def process_outputs(self, windowed_input_element, results): File "apache_beam/runners/common.py", line 847, in apache_beam.runners.common._OutputProcessor.process_outputs self.main_receivers.receive(windowed_value) File "apache_beam/runners/worker/operations.py", line 131, in apache_beam.runners.worker.operations.SingletonConsumerSet.receive self.consumer.process(windowed_value) File "apache_beam/runners/worker/operations.py", line 537, in apache_beam.runners.worker.operations.DoOperation.process with self.scoped_process_state: File "apache_beam/runners/worker/operations.py", line 538, in apache_beam.runners.worker.operations.DoOperation.process delayed_application = self.dofn_receiver.receive(o) File "apache_beam/runners/common.py", line 723, in apache_beam.runners.common.DoFnRunner.receive self.process(windowed_value) File "apache_beam/runners/common.py", line 729, in apache_beam.runners.common.DoFnRunner.process self._reraise_augmented(exn) File "apache_beam/runners/common.py", line 777, in apache_beam.runners.common.DoFnRunner._reraise_augmented raise_with_traceback(new_exn) File "apache_beam/runners/common.py", line 727, in apache_beam.runners.common.DoFnRunner.process return self.do_fn_invoker.invoke_process(windowed_value) File "apache_beam/runners/common.py", line 419, in apache_beam.runners.common.SimpleInvoker.invoke_process windowed_value, self.process_method(windowed_value.value)) File "streaming_twitter.py", line 188, in <lambda> File "streaming_twitter.py", line 102, in estimate File "streaming_twitter.py", line 84, in estimate_aiplatform File "streaming_twitter.py", line 42, in get_service File "/usr/local/lib/python2.7/dist-packages/googleapiclient/_helpers.py", line 130, in positional_wrapper return wrapped(*args, **kwargs) File "/usr/local/lib/python2.7/dist-packages/googleapiclient/discovery.py", line 227, in build credentials=credentials) File "/usr/local/lib/python2.7/dist-packages/googleapiclient/_helpers.py", line 130, in positional_wrapper return wrapped(*args, **kwargs) File "/usr/local/lib/python2.7/dist-packages/googleapiclient/discovery.py", line 363, in build_from_document credentials = _auth.default_credentials() File "/usr/local/lib/python2.7/dist-packages/googleapiclient/_auth.py", line 42, in default_credentials credentials, _ = google.auth.default() File "/usr/local/lib/python2.7/dist-packages/google/auth/_default.py", line 306, in default raise exceptions.DefaultCredentialsError(_HELP_MESSAGE) DefaultCredentialsError: Could not automatically determine credentials. Please set GOOGLE_APPLICATION_CREDENTIALS or explicitly create credentials and re-run the application. For more information, please see https://cloud.google.com/docs/authentication/getting-started [while running 'generatedPtransform-450797'] org.apache.beam.runners.fnexecution.control.FnApiControlClient$ResponseStreamObserver.onNext(FnApiControlClient.java:157) org.apache.beam.runners.fnexecution.control.FnApiControlClient$ResponseStreamObserver.onNext(FnApiControlClient.java:140) org.apache.beam.vendor.grpc.v1p13p1.io.grpc.stub.ServerCalls$StreamingServerCallHandler$StreamingServerCallListener.onMessage(ServerCalls.java:248) org.apache.beam.vendor.grpc.v1p13p1.io.grpc.ForwardingServerCallListener.onMessage(ForwardingServerCallListener.java:33) org.apache.beam.vendor.grpc.v1p13p1.io.grpc.Contexts$ContextualizedServerCallListener.onMessage(Contexts.java:76) org.apache.beam.vendor.grpc.v1p13p1.io.grpc.internal.ServerCallImpl$ServerStreamListenerImpl.messagesAvailable(ServerCallImpl.java:263) org.apache.beam.vendor.grpc.v1p13p1.io.grpc.internal.ServerImpl$JumpToApplicationThreadServerStreamListener$1MessagesAvailable.runInContext(ServerImpl.java:683) org.apache.beam.vendor.grpc.v1p13p1.io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) org.apache.beam.vendor.grpc.v1p13p1.io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:123) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) java.lang.Thread.run(Thread.java:745)