Я пытаюсь загрузить только самые последние файлы .csv из моего хранилища S3 и сталкиваюсь с ошибкой, которая говорит "TypeError: expected string or bytes-like object."
В настоящее время у меня есть рабочий код, который идентифицирует последние измененные объекты S3, сортирует эти объекты и помещает их в список с именем latest_files.
session = boto3.Session()
s3_resource = boto3.resource('s3')
my_bucket = s3_resource.Bucket('chansbucket')
get_last_modified = lambda obj: int(obj.last_modified.strftime('%s'))
unsorted = []
# filters through the bucket and appends objects to the unsorted list
for file in my_bucket.objects.filter():
unsorted.append(file)
# sorts last five files in unsorted by last modified time
latest_files = [obj.key for obj in sorted(unsorted, key=get_last_modified, reverse=True)][0:5]
Теперь я хочу просмотреть latest_files и загрузить только те, которые заканчиваются на .csv.
for file in latest_files:
if file.endswith('.csv'):
s3_resource.meta.client.download_file(my_bucket, file, '/Users/mikechan/projects/TT_product_analyses/raw_csv_files/' + file)
Здесь я получаю сообщение об ошибке TypeError: expected string or bytes-like object
Вот трассировка:
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-27-ca90c5ad9c53> in <module>()
1 for file in latest_files:
2 if file.endswith('.csv'):
----> 3 s3_resource.meta.client.download_file(my_bucket, str(file), '/Users/mikechan/projects/TT_product_analyses/raw_csv_files/' + str(file))
4
5
~/anaconda/lib/python3.6/site-packages/boto3/s3/inject.py in download_file(self, Bucket, Key, Filename, ExtraArgs, Callback, Config)
170 return transfer.download_file(
171 bucket=Bucket, key=Key, filename=Filename,
--> 172 extra_args=ExtraArgs, callback=Callback)
173
174
~/anaconda/lib/python3.6/site-packages/boto3/s3/transfer.py in download_file(self, bucket, key, filename, extra_args, callback)
305 bucket, key, filename, extra_args, subscribers)
306 try:
--> 307 future.result()
308 # This is for backwards compatibility where when retries are
309 # exceeded we need to throw the same error from boto3 instead of
~/anaconda/lib/python3.6/site-packages/s3transfer/futures.py in result(self)
71 # however if a KeyboardInterrupt is raised we want want to exit
72 # out of this and propogate the exception.
---> 73 return self._coordinator.result()
74 except KeyboardInterrupt as e:
75 self.cancel()
~/anaconda/lib/python3.6/site-packages/s3transfer/futures.py in result(self)
231 # final result.
232 if self._exception:
--> 233 raise self._exception
234 return self._result
235
~/anaconda/lib/python3.6/site-packages/s3transfer/tasks.py in _main(self, transfer_future, **kwargs)
253 # Call the submit method to start submitting tasks to execute the
254 # transfer.
--> 255 self._submit(transfer_future=transfer_future, **kwargs)
256 except BaseException as e:
257 # If there was an exception raised during the submission of task
~/anaconda/lib/python3.6/site-packages/s3transfer/download.py in _submit(self, client, config, osutil, request_executor, io_executor, transfer_future, bandwidth_limiter)
351 Bucket=transfer_future.meta.call_args.bucket,
352 Key=transfer_future.meta.call_args.key,
--> 353 **transfer_future.meta.call_args.extra_args
354 )
355 transfer_future.meta.provide_transfer_size(
~/.local/lib/python3.6/site-packages/botocore/client.py in _api_call(self, *args, **kwargs)
318 "%s() only accepts keyword arguments." % py_operation_name)
319 # The "self" in this scope is referring to the BaseClient.
--> 320 return self._make_api_call(operation_name, kwargs)
321
322 _api_call.__name__ = str(py_operation_name)
~/.local/lib/python3.6/site-packages/botocore/client.py in _make_api_call(self, operation_name, api_params)
594 }
595 request_dict = self._convert_to_request_dict(
--> 596 api_params, operation_model, context=request_context)
597
598 service_id = self._service_model.service_id.hyphenize()
~/.local/lib/python3.6/site-packages/botocore/client.py in _convert_to_request_dict(self, api_params, operation_model, context)
628 context=None):
629 api_params = self._emit_api_params(
--> 630 api_params, operation_model, context)
631 request_dict = self._serializer.serialize_to_request(
632 api_params, operation_model)
~/.local/lib/python3.6/site-packages/botocore/client.py in _emit_api_params(self, api_params, operation_model, context)
658 service_id=service_id,
659 operation_name=operation_name),
--> 660 params=api_params, model=operation_model, context=context)
661 return api_params
662
~/.local/lib/python3.6/site-packages/botocore/hooks.py in emit(self, event_name, **kwargs)
354 def emit(self, event_name, **kwargs):
355 aliased_event_name = self._alias_event_name(event_name)
--> 356 return self._emitter.emit(aliased_event_name, **kwargs)
357
358 def emit_until_response(self, event_name, **kwargs):
~/.local/lib/python3.6/site-packages/botocore/hooks.py in emit(self, event_name, **kwargs)
226 handlers.
227 """
--> 228 return self._emit(event_name, kwargs)
229
230 def emit_until_response(self, event_name, **kwargs):
~/.local/lib/python3.6/site-packages/botocore/hooks.py in _emit(self, event_name, kwargs, stop_on_response)
209 for handler in handlers_to_call:
210 logger.debug('Event %s: calling handler %s', event_name, handler)
--> 211 response = handler(**kwargs)
212 responses.append((handler, response))
213 if stop_on_response and response is not None:
~/.local/lib/python3.6/site-packages/botocore/handlers.py in validate_bucket_name(params, **kwargs)
216 return
217 bucket = params['Bucket']
--> 218 if VALID_BUCKET.search(bucket) is None:
219 error_msg = (
220 'Invalid bucket name "%s": Bucket name must match '
TypeError: expected string or bytes-like object
Вы можете помочь? Я чувствую, что это что-то довольно простое, но я абсолютный новичок и постоянно стучу головой об свой стол об этом. Любая помощь приветствуется.
Спасибо!