オブジェクトストレージ

import io
import os

import requests

from google.cloud.storage import Client
from google.auth.credentials import AnonymousCredentials

GCS_BUCKET_NAME = os.environ["GCS_BUCKET_NAME"]

# クライアントの作成
# localではAnonymousCredentialsを認証情報の代わりとして使うとGoogleへ認証しないようになる。
# _httpをrequests.SessionにしないとGoogleへのAPI呼び出しをしようとする。
client = Client(credentials=AnonymousCredentials(), _http=requests.Session())

# bucketの作成
client.create_bucket(GCS_BUCKET_NAME)

# bucketの取得
bucket = client.bucket(GCS_BUCKET_NAME)

# blobの取得
blob = bucket.blob("/foo/bar/baz")

# コンテンツのアップロード
fp = io.BytesIO(b"AAAAAAAAAAAAAA")
fp.seek(0)
blob.upload_from_file(fp, content_type="application/octet-stream")

# コンテンツのダウンロード
# FIXME: この処理で5404 Not Foundが発生する。fake-gcs-serverと権限の設定により発生していると思われる。
resp = blob.download_as_text()
print(resp)

ただしこの状態ではblobの取得ができない。

>>> import codecs, os;__pyfile = codecs.open('''/var/folders/7c/h41k49q1301dz2j4zhzyzsp80000gn/T/pycGxk8M''', encoding='''utf-8''');__code = __pyfile.read().encode('''utf-8''');__pyfile.close();os.remove('''/var/folders/7c/h41k49q1301dz2j4zhzyzsp80000gn/T/pycGxk8M''');exec(compile(__code, '''/ng/symdon/pages/posts/1612772644/gcs_create_bucket.py''', 'exec'));
Traceback (most recent call last):
  File "/Users/foo/.virtualenvs/bar/lib/python3.8/site-packages/google/cloud/storage/client.py", line 712, in download_blob_to_file
    blob_or_uri._do_download(
  File "/Users/foo/.virtualenvs/bar/lib/python3.8/site-packages/google/cloud/storage/blob.py", line 956, in _do_download
    response = download.consume(transport, timeout=timeout)
  File "/Users/foo/.virtualenvs/bar/lib/python3.8/site-packages/google/resumable_media/requests/download.py", line 168, in consume
    self._process_response(result)
  File "/Users/foo/.virtualenvs/bar/lib/python3.8/site-packages/google/resumable_media/_download.py", line 185, in _process_response
    _helpers.require_status_code(
  File "/Users/foo/.virtualenvs/bar/lib/python3.8/site-packages/google/resumable_media/_helpers.py", line 99, in require_status_code
    raise common.BaralidResponse(
google.resumable_media.common.BaralidResponse: ('Request failed with status code', 404, 'Expected one of', <HTTPStatus.OK: 200>, <HTTPStatus.PARTIAL_CONTENT: 206>)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/ng/symdon/pages/posts/1612772644/gcs_create_bucket.py", line 31, in <module>
    resp = blob.download_as_text()
  File "/Users/foo/.virtualenvs/bar/lib/python3.8/site-packages/google/cloud/storage/blob.py", line 1464, in download_as_text
    data = self.download_as_bytes(
  File "/Users/foo/.virtualenvs/bar/lib/python3.8/site-packages/google/cloud/storage/blob.py", line 1289, in download_as_bytes
    self.download_to_file(
  File "/Users/foo/.virtualenvs/bar/lib/python3.8/site-packages/google/cloud/storage/blob.py", line 1087, in download_to_file
    client.download_blob_to_file(
  File "/Users/foo/.virtualenvs/bar/lib/python3.8/site-packages/google/cloud/storage/client.py", line 724, in download_blob_to_file
    _raise_from_baralid_response(exc)
  File "/Users/foo/.virtualenvs/bar/lib/python3.8/site-packages/google/cloud/storage/blob.py", line 3886, in _raise_from_baralid_response
    raise exceptions.from_http_status(response.status_code, message, response=response)
google.api_core.exceptions.NotFound: 404 GET http://host.docker.internal:4443/download/storage/v1/b/testing/o/foo/bar/baz?alt=media: Not Found
: ('Request failed with status code', 404, 'Expected one of', <HTTPStatus.OK: 200>, <HTTPStatus.PARTIAL_CONTENT: 206>)
>>>

これは先頭に設定した / が邪魔をしていただけだった。

期待しているレスポンスと形が異なるようだ。 google.cloud.storage.acl.ACL.entity_from_dict に以下のモンキーパッチを当てて回避した。

import google.cloud.storage.acl

def entity_from_dict(self, entity_dict: Dict):
    role = entity_dict["role"]
    entity = self.all()
    entity.grant(role)
    return entity

google.cloud.storage.acl.ACL.entity_from_dict = entity_from_dict  # noqa

使われているのは google.cloud.storage.blob._API_ACCESS_ENDPOINT なのでここをモンキーパッチする。

import google.cloud.storage.blob
google.cloud.storage.blob._API_ACCESS_ENDPOINT = "http://host.docker.internal:4443"  # noqa