blob: a01b56515737ab72d232f59752272e863da41f4c [file] [log] [blame]
Copybara854996b2021-09-07 19:36:02 +00001# Copyright 2016 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style
3# license that can be found in the LICENSE file or at
4# https://developers.google.com/open-source/licenses/bsd
5
6"""Set of helpers for interacting with Google Cloud Storage."""
7from __future__ import print_function
8from __future__ import division
9from __future__ import absolute_import
10
11import base64
12import logging
13import os
14import time
15import urllib
16import uuid
17
18from datetime import datetime, timedelta
19
20from google.appengine.api import app_identity
21from google.appengine.api import images
22from google.appengine.api import memcache
23from google.appengine.api import urlfetch
24from third_party import cloudstorage
25from third_party.cloudstorage import errors
26
27from framework import filecontent
28from framework import framework_constants
29from framework import framework_helpers
30
31
32ATTACHMENT_TTL = timedelta(seconds=30)
33
34IS_DEV_APPSERVER = (
35 'development' in os.environ.get('SERVER_SOFTWARE', '').lower())
36
37RESIZABLE_MIME_TYPES = [
38 'image/png', 'image/jpg', 'image/jpeg', 'image/gif', 'image/webp',
39 ]
40
41DEFAULT_THUMB_WIDTH = 250
42DEFAULT_THUMB_HEIGHT = 200
43LOGO_THUMB_WIDTH = 110
44LOGO_THUMB_HEIGHT = 30
45MAX_ATTACH_SIZE_TO_COPY = 10 * 1024 * 1024 # 10 MB
46# GCS signatures are valid for 10 minutes by default, but cache them for
47# 5 minutes just to be on the safe side.
48GCS_SIG_TTL = 60 * 5
49
50
51def _Now():
52 return datetime.utcnow()
53
54
55class UnsupportedMimeType(Exception):
56 pass
57
58
59def DeleteObjectFromGCS(object_id):
60 object_path = ('/' + app_identity.get_default_gcs_bucket_name() + object_id)
61 cloudstorage.delete(object_path)
62
63
64def StoreObjectInGCS(
65 content, mime_type, project_id, thumb_width=DEFAULT_THUMB_WIDTH,
66 thumb_height=DEFAULT_THUMB_HEIGHT, filename=None):
67 bucket_name = app_identity.get_default_gcs_bucket_name()
68 guid = uuid.uuid4()
69 object_id = '/%s/attachments/%s' % (project_id, guid)
70 object_path = '/' + bucket_name + object_id
71 options = {}
72 if filename:
73 if not framework_constants.FILENAME_RE.match(filename):
74 logging.info('bad file name: %s' % filename)
75 filename = 'attachment.dat'
76 options['Content-Disposition'] = 'inline; filename="%s"' % filename
77 logging.info('Writing with options %r', options)
78 with cloudstorage.open(object_path, 'w', mime_type, options=options) as f:
79 f.write(content)
80
81 if mime_type in RESIZABLE_MIME_TYPES:
82 # Create and save a thumbnail too.
83 thumb_content = None
84 try:
85 thumb_content = images.resize(content, thumb_width, thumb_height)
86 except images.LargeImageError:
87 # Don't log the whole exception because we don't need to see
88 # this on the Cloud Error Reporting page.
89 logging.info('Got LargeImageError on image with %d bytes', len(content))
90 except Exception, e:
91 # Do not raise exception for incorrectly formed images.
92 # See https://bugs.chromium.org/p/monorail/issues/detail?id=597 for more
93 # detail.
94 logging.exception(e)
95 if thumb_content:
96 thumb_path = '%s-thumbnail' % object_path
97 with cloudstorage.open(thumb_path, 'w', 'image/png') as f:
98 f.write(thumb_content)
99
100 return object_id
101
102
103def CheckMimeTypeResizable(mime_type):
104 if mime_type not in RESIZABLE_MIME_TYPES:
105 raise UnsupportedMimeType(
106 'Please upload a logo with one of the following mime types:\n%s' %
107 ', '.join(RESIZABLE_MIME_TYPES))
108
109
110def StoreLogoInGCS(file_name, content, project_id):
111 mime_type = filecontent.GuessContentTypeFromFilename(file_name)
112 CheckMimeTypeResizable(mime_type)
113 if '\\' in file_name: # IE insists on giving us the whole path.
114 file_name = file_name[file_name.rindex('\\') + 1:]
115 return StoreObjectInGCS(
116 content, mime_type, project_id, thumb_width=LOGO_THUMB_WIDTH,
117 thumb_height=LOGO_THUMB_HEIGHT)
118
119
120@framework_helpers.retry(3, delay=0.25, backoff=1.25)
121def _FetchSignedURL(url):
122 """Request that devstorage API signs a GCS content URL."""
123 resp = urlfetch.fetch(url, follow_redirects=False)
124 redir = resp.headers["Location"]
125 return redir
126
127
128def SignUrl(bucket, object_id):
129 """Get a signed URL to download a GCS object.
130
131 Args:
132 bucket: string name of the GCS bucket.
133 object_id: string object ID of the file within that bucket.
134
135 Returns:
136 A signed URL, or '/mising-gcs-url' if signing failed.
137 """
138 try:
139 cache_key = 'gcs-object-url-%s' % object_id
140 cached = memcache.get(key=cache_key)
141 if cached is not None:
142 return cached
143
144 if IS_DEV_APPSERVER:
145 attachment_url = '/_ah/gcs/%s%s' % (bucket, object_id)
146 else:
147 result = ('https://www.googleapis.com/storage/v1/b/'
148 '{bucket}/o/{object_id}?access_token={token}&alt=media')
149 scopes = ['https://www.googleapis.com/auth/devstorage.read_only']
150 if object_id[0] == '/':
151 object_id = object_id[1:]
152 url = result.format(
153 bucket=bucket,
154 object_id=urllib.quote_plus(object_id),
155 token=app_identity.get_access_token(scopes)[0])
156 attachment_url = _FetchSignedURL(url)
157
158 if not memcache.set(key=cache_key, value=attachment_url, time=GCS_SIG_TTL):
159 logging.error('Could not cache gcs url %s for %s', attachment_url,
160 object_id)
161
162 return attachment_url
163
164 except Exception as e:
165 logging.exception(e)
166 return '/missing-gcs-url'
167
168
169def MaybeCreateDownload(bucket_name, object_id, filename):
170 """If the obj is not huge, and no download version exists, create it."""
171 src = '/%s%s' % (bucket_name, object_id)
172 dst = '/%s%s-download' % (bucket_name, object_id)
173 cloudstorage.validate_file_path(src)
174 cloudstorage.validate_file_path(dst)
175 logging.info('Maybe create %r from %r', dst, src)
176
177 if IS_DEV_APPSERVER:
178 logging.info('dev environment never makes download copies.')
179 return False
180
181 # If "Download" object already exists, we are done.
182 try:
183 cloudstorage.stat(dst)
184 logging.info('Download version of attachment already exists')
185 return True
186 except errors.NotFoundError:
187 pass
188
189 # If "View" object is huge, give up.
190 src_stat = cloudstorage.stat(src)
191 if src_stat.st_size > MAX_ATTACH_SIZE_TO_COPY:
192 logging.info('Download version of attachment would be too big')
193 return False
194
195 with cloudstorage.open(src, 'r') as infile:
196 content = infile.read()
197 logging.info('opened GCS object and read %r bytes', len(content))
198 content_type = src_stat.content_type
199 options = {
200 'Content-Disposition': 'attachment; filename="%s"' % filename,
201 }
202 logging.info('Writing with options %r', options)
203 with cloudstorage.open(dst, 'w', content_type, options=options) as outfile:
204 outfile.write(content)
205 logging.info('done writing')
206
207 return True