Merge branch 'main' into avm99963-monorail
Merged commit 34d8229ae2b51fb1a15bd208e6fe6185c94f6266
GitOrigin-RevId: 7ee0917f93a577e475f8e09526dd144d245593f4
diff --git a/third_party/endpoints/_endpointscfg_impl.py b/third_party/endpoints/_endpointscfg_impl.py
new file mode 100644
index 0000000..2d3f740
--- /dev/null
+++ b/third_party/endpoints/_endpointscfg_impl.py
@@ -0,0 +1,617 @@
+#!/usr/bin/python
+# Copyright 2017 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+r"""External script for generating Cloud Endpoints related files.
+
+The gen_discovery_doc subcommand takes a list of fully qualified ProtoRPC
+service names and calls a cloud service which generates a discovery document in
+REST or RPC style.
+
+Example:
+ endpointscfg.py gen_discovery_doc -o . -f rest postservice.GreetingsV1
+
+The gen_client_lib subcommand takes a discovery document and calls a cloud
+service to generate a client library for a target language (currently just Java)
+
+Example:
+ endpointscfg.py gen_client_lib java -o . greetings-v0.1.discovery
+
+The get_client_lib subcommand does both of the above commands at once.
+
+Example:
+ endpointscfg.py get_client_lib java -o . postservice.GreetingsV1
+
+The gen_api_config command outputs an .api configuration file for a service.
+
+Example:
+ endpointscfg.py gen_api_config -o . -a /path/to/app \
+ --hostname myhost.appspot.com postservice.GreetingsV1
+"""
+
+from __future__ import absolute_import
+
+import argparse
+import collections
+import contextlib
+import logging
+import os
+import re
+import sys
+from six.moves import urllib
+import urllib2
+
+import yaml
+from google.appengine.ext import testbed
+
+from . import api_config
+from . import discovery_generator
+from . import openapi_generator
+from . import remote
+
+# Conditional import, pylint: disable=g-import-not-at-top
+try:
+ import json
+except ImportError:
+ # If we can't find json packaged with Python import simplejson, which is
+ # packaged with the SDK.
+ import simplejson as json
+
+
+CLIENT_LIBRARY_BASE = 'https://google-api-client-libraries.appspot.com/generate'
+_VISIBLE_COMMANDS = ('get_client_lib', 'get_discovery_doc', 'get_openapi_spec')
+
+
+class ServerRequestException(Exception):
+ """Exception for problems with the request to a server."""
+
+ def __init__(self, http_error):
+ """Create a ServerRequestException from a given urllib2.HTTPError.
+
+ Args:
+ http_error: The HTTPError that the ServerRequestException will be
+ based on.
+ """
+ error_details = None
+ error_response = None
+ if http_error.fp:
+ try:
+ error_response = http_error.fp.read()
+ error_body = json.loads(error_response)
+ error_details = ['%s: %s' % (detail['message'], detail['debug_info'])
+ for detail in error_body['error']['errors']]
+ except (ValueError, TypeError, KeyError):
+ pass
+ if error_details:
+ error_details_str = ', '.join(error_details)
+ error_message = ('HTTP %s (%s) error when communicating with URL: %s. '
+ 'Details: %s' % (http_error.code, http_error.reason,
+ http_error.filename, error_details_str))
+ else:
+ error_message = ('HTTP %s (%s) error when communicating with URL: %s. '
+ 'Response: %s' % (http_error.code, http_error.reason,
+ http_error.filename,
+ error_response))
+ super(ServerRequestException, self).__init__(error_message)
+
+
+class _EndpointsParser(argparse.ArgumentParser):
+ """Create a subclass of argparse.ArgumentParser for Endpoints."""
+
+ def error(self, message):
+ """Override superclass to support customized error message.
+
+ Error message needs to be rewritten in order to display visible commands
+ only, when invalid command is called by user. Otherwise, hidden commands
+ will be displayed in stderr, which is not expected.
+
+ Refer the following argparse python documentation for detailed method
+ information:
+ http://docs.python.org/2/library/argparse.html#exiting-methods
+
+ Args:
+ message: original error message that will be printed to stderr
+ """
+ # subcommands_quoted is the same as subcommands, except each value is
+ # surrounded with double quotes. This is done to match the standard
+ # output of the ArgumentParser, while hiding commands we don't want users
+ # to use, as they are no longer documented and only here for legacy use.
+ subcommands_quoted = ', '.join(
+ [repr(command) for command in _VISIBLE_COMMANDS])
+ subcommands = ', '.join(_VISIBLE_COMMANDS)
+ message = re.sub(
+ r'(argument {%s}: invalid choice: .*) \(choose from (.*)\)$'
+ % subcommands, r'\1 (choose from %s)' % subcommands_quoted, message)
+ super(_EndpointsParser, self).error(message)
+
+
+def _WriteFile(output_path, name, content):
+ """Write given content to a file in a given directory.
+
+ Args:
+ output_path: The directory to store the file in.
+ name: The name of the file to store the content in.
+ content: The content to write to the file.close
+
+ Returns:
+ The full path to the written file.
+ """
+ path = os.path.join(output_path, name)
+ with open(path, 'wb') as f:
+ f.write(content)
+ return path
+
+
+def GenApiConfig(service_class_names, config_string_generator=None,
+ hostname=None, application_path=None, **additional_kwargs):
+ """Write an API configuration for endpoints annotated ProtoRPC services.
+
+ Args:
+ service_class_names: A list of fully qualified ProtoRPC service classes.
+ config_string_generator: A generator object that produces API config strings
+ using its pretty_print_config_to_json method.
+ hostname: A string hostname which will be used as the default version
+ hostname. If no hostname is specificied in the @endpoints.api decorator,
+ this value is the fallback.
+ application_path: A string with the path to the AppEngine application.
+
+ Raises:
+ TypeError: If any service classes don't inherit from remote.Service.
+ messages.DefinitionNotFoundError: If a service can't be found.
+
+ Returns:
+ A map from service names to a string containing the API configuration of the
+ service in JSON format.
+ """
+ # First, gather together all the different APIs implemented by these
+ # classes. There may be fewer APIs than service classes. Each API is
+ # uniquely identified by (name, version). Order needs to be preserved here,
+ # so APIs that were listed first are returned first.
+ api_service_map = collections.OrderedDict()
+ resolved_services = []
+
+ for service_class_name in service_class_names:
+ module_name, base_service_class_name = service_class_name.rsplit('.', 1)
+ module = __import__(module_name, fromlist=base_service_class_name)
+ service = getattr(module, base_service_class_name)
+ if hasattr(service, 'get_api_classes'):
+ resolved_services.extend(service.get_api_classes())
+ elif (not isinstance(service, type) or
+ not issubclass(service, remote.Service)):
+ raise TypeError('%s is not a ProtoRPC service' % service_class_name)
+ else:
+ resolved_services.append(service)
+
+ for resolved_service in resolved_services:
+ services = api_service_map.setdefault(
+ (resolved_service.api_info.name, resolved_service.api_info.api_version), [])
+ services.append(resolved_service)
+
+ # If hostname isn't specified in the API or on the command line, we'll
+ # try to build it from information in app.yaml.
+ app_yaml_hostname = _GetAppYamlHostname(application_path)
+
+ service_map = collections.OrderedDict()
+ config_string_generator = (
+ config_string_generator or api_config.ApiConfigGenerator())
+ for api_info, services in api_service_map.items():
+ assert services, 'An API must have at least one ProtoRPC service'
+ # Only override hostname if None. Hostname will be the same for all
+ # services within an API, since it's stored in common info.
+ hostname = services[0].api_info.hostname or hostname or app_yaml_hostname
+
+ # Map each API by name-version.
+ service_map['%s-%s' % api_info] = (
+ config_string_generator.pretty_print_config_to_json(
+ services, hostname=hostname, **additional_kwargs))
+
+ return service_map
+
+
+def _GetAppYamlHostname(application_path, open_func=open):
+ """Build the hostname for this app based on the name in app.yaml.
+
+ Args:
+ application_path: A string with the path to the AppEngine application. This
+ should be the directory containing the app.yaml file.
+ open_func: Function to call to open a file. Used to override the default
+ open function in unit tests.
+
+ Returns:
+ A hostname, usually in the form of "myapp.appspot.com", based on the
+ application name in the app.yaml file. If the file can't be found or
+ there's a problem building the name, this will return None.
+ """
+ try:
+ app_yaml_file = open_func(os.path.join(application_path or '.', 'app.yaml'))
+ config = yaml.safe_load(app_yaml_file.read())
+ except IOError:
+ # Couldn't open/read app.yaml.
+ return None
+
+ application = config.get('application')
+ if not application:
+ return None
+
+ if ':' in application:
+ # Don't try to deal with alternate domains.
+ return None
+
+ # If there's a prefix ending in a '~', strip it.
+ tilde_index = application.rfind('~')
+ if tilde_index >= 0:
+ application = application[tilde_index + 1:]
+ if not application:
+ return None
+
+ return '%s.appspot.com' % application
+
+
+def _GenDiscoveryDoc(service_class_names,
+ output_path, hostname=None,
+ application_path=None):
+ """Write discovery documents generated from the service classes to file.
+
+ Args:
+ service_class_names: A list of fully qualified ProtoRPC service names.
+ output_path: The directory to output the discovery docs to.
+ hostname: A string hostname which will be used as the default version
+ hostname. If no hostname is specificied in the @endpoints.api decorator,
+ this value is the fallback. Defaults to None.
+ application_path: A string containing the path to the AppEngine app.
+
+ Returns:
+ A list of discovery doc filenames.
+ """
+ output_files = []
+ service_configs = GenApiConfig(
+ service_class_names, hostname=hostname,
+ config_string_generator=discovery_generator.DiscoveryGenerator(),
+ application_path=application_path)
+ for api_name_version, config in service_configs.items():
+ discovery_name = api_name_version + '.discovery'
+ output_files.append(_WriteFile(output_path, discovery_name, config))
+
+ return output_files
+
+
+def _GenOpenApiSpec(service_class_names, output_path, hostname=None,
+ application_path=None, x_google_api_name=False):
+ """Write openapi documents generated from the service classes to file.
+
+ Args:
+ service_class_names: A list of fully qualified ProtoRPC service names.
+ output_path: The directory to which to output the OpenAPI specs.
+ hostname: A string hostname which will be used as the default version
+ hostname. If no hostname is specified in the @endpoints.api decorator,
+ this value is the fallback. Defaults to None.
+ application_path: A string containing the path to the AppEngine app.
+
+ Returns:
+ A list of OpenAPI spec filenames.
+ """
+ output_files = []
+ service_configs = GenApiConfig(
+ service_class_names, hostname=hostname,
+ config_string_generator=openapi_generator.OpenApiGenerator(),
+ application_path=application_path,
+ x_google_api_name=x_google_api_name)
+ for api_name_version, config in service_configs.items():
+ openapi_name = api_name_version.replace('-', '') + 'openapi.json'
+ output_files.append(_WriteFile(output_path, openapi_name, config))
+
+ return output_files
+
+
+def _GenClientLib(discovery_path, language, output_path, build_system):
+ """Write a client library from a discovery doc.
+
+ Args:
+ discovery_path: Path to the discovery doc used to generate the client
+ library.
+ language: The client library language to generate. (java)
+ output_path: The directory to output the client library zip to.
+ build_system: The target build system for the client library language.
+
+ Raises:
+ IOError: If reading the discovery doc fails.
+ ServerRequestException: If fetching the generated client library fails.
+
+ Returns:
+ The path to the zipped client library.
+ """
+ with open(discovery_path) as f:
+ discovery_doc = f.read()
+
+ client_name = re.sub(r'\.discovery$', '.zip',
+ os.path.basename(discovery_path))
+
+ return _GenClientLibFromContents(discovery_doc, language, output_path,
+ build_system, client_name)
+
+
+def _GenClientLibFromContents(discovery_doc, language, output_path,
+ build_system, client_name):
+ """Write a client library from a discovery doc.
+
+ Args:
+ discovery_doc: A string, the contents of the discovery doc used to
+ generate the client library.
+ language: A string, the client library language to generate. (java)
+ output_path: A string, the directory to output the client library zip to.
+ build_system: A string, the target build system for the client language.
+ client_name: A string, the filename used to save the client lib.
+
+ Raises:
+ IOError: If reading the discovery doc fails.
+ ServerRequestException: If fetching the generated client library fails.
+
+ Returns:
+ The path to the zipped client library.
+ """
+
+ body = urllib.parse.urlencode({'lang': language, 'content': discovery_doc,
+ 'layout': build_system})
+ request = urllib2.Request(CLIENT_LIBRARY_BASE, body)
+ try:
+ with contextlib.closing(urllib2.urlopen(request)) as response:
+ content = response.read()
+ return _WriteFile(output_path, client_name, content)
+ except urllib2.HTTPError as error:
+ raise ServerRequestException(error)
+
+
+def _GetClientLib(service_class_names, language, output_path, build_system,
+ hostname=None, application_path=None):
+ """Fetch client libraries from a cloud service.
+
+ Args:
+ service_class_names: A list of fully qualified ProtoRPC service names.
+ language: The client library language to generate. (java)
+ output_path: The directory to output the discovery docs to.
+ build_system: The target build system for the client library language.
+ hostname: A string hostname which will be used as the default version
+ hostname. If no hostname is specificied in the @endpoints.api decorator,
+ this value is the fallback. Defaults to None.
+ application_path: A string containing the path to the AppEngine app.
+
+ Returns:
+ A list of paths to client libraries.
+ """
+ client_libs = []
+ service_configs = GenApiConfig(
+ service_class_names, hostname=hostname,
+ config_string_generator=discovery_generator.DiscoveryGenerator(),
+ application_path=application_path)
+ for api_name_version, config in service_configs.items():
+ client_name = api_name_version + '.zip'
+ client_libs.append(
+ _GenClientLibFromContents(config, language, output_path,
+ build_system, client_name))
+ return client_libs
+
+
+def _GenApiConfigCallback(args, api_func=GenApiConfig):
+ """Generate an api file.
+
+ Args:
+ args: An argparse.Namespace object to extract parameters from.
+ api_func: A function that generates and returns an API configuration
+ for a list of services.
+ """
+ service_configs = api_func(args.service,
+ hostname=args.hostname,
+ application_path=args.application)
+
+ for api_name_version, config in service_configs.items():
+ _WriteFile(args.output, api_name_version + '.api', config)
+
+
+def _GetClientLibCallback(args, client_func=_GetClientLib):
+ """Generate discovery docs and client libraries to files.
+
+ Args:
+ args: An argparse.Namespace object to extract parameters from.
+ client_func: A function that generates client libraries and stores them to
+ files, accepting a list of service names, a client library language,
+ an output directory, a build system for the client library language, and
+ a hostname.
+ """
+ client_paths = client_func(
+ args.service, args.language, args.output, args.build_system,
+ hostname=args.hostname, application_path=args.application)
+
+ for client_path in client_paths:
+ print 'API client library written to %s' % client_path
+
+
+def _GenDiscoveryDocCallback(args, discovery_func=_GenDiscoveryDoc):
+ """Generate discovery docs to files.
+
+ Args:
+ args: An argparse.Namespace object to extract parameters from
+ discovery_func: A function that generates discovery docs and stores them to
+ files, accepting a list of service names, a discovery doc format, and an
+ output directory.
+ """
+ discovery_paths = discovery_func(args.service, args.output,
+ hostname=args.hostname,
+ application_path=args.application)
+ for discovery_path in discovery_paths:
+ print 'API discovery document written to %s' % discovery_path
+
+
+def _GenOpenApiSpecCallback(args, openapi_func=_GenOpenApiSpec):
+ """Generate OpenAPI (Swagger) specs to files.
+
+ Args:
+ args: An argparse.Namespace object to extract parameters from
+ openapi_func: A function that generates OpenAPI specs and stores them to
+ files, accepting a list of service names and an output directory.
+ """
+ openapi_paths = openapi_func(args.service, args.output,
+ hostname=args.hostname,
+ application_path=args.application,
+ x_google_api_name=args.x_google_api_name)
+ for openapi_path in openapi_paths:
+ print 'OpenAPI spec written to %s' % openapi_path
+
+
+def _GenClientLibCallback(args, client_func=_GenClientLib):
+ """Generate a client library to file.
+
+ Args:
+ args: An argparse.Namespace object to extract parameters from
+ client_func: A function that generates client libraries and stores them to
+ files, accepting a path to a discovery doc, a client library language, an
+ output directory, and a build system for the client library language.
+ """
+ client_path = client_func(args.discovery_doc[0], args.language, args.output,
+ args.build_system)
+ print 'API client library written to %s' % client_path
+
+
+def MakeParser(prog):
+ """Create an argument parser.
+
+ Args:
+ prog: The name of the program to use when outputting help text.
+
+ Returns:
+ An argparse.ArgumentParser built to specification.
+ """
+
+ def AddStandardOptions(parser, *args):
+ """Add common endpoints options to a parser.
+
+ Args:
+ parser: The parser to add options to.
+ *args: A list of option names to add. Possible names are: application,
+ format, output, language, service, and discovery_doc.
+ """
+ if 'application' in args:
+ parser.add_argument('-a', '--application', default='.',
+ help='The path to the Python App Engine App')
+ if 'format' in args:
+ # This used to be a valid option, allowing the user to select 'rest' or 'rpc',
+ # but now 'rest' is the only valid type. The argument remains so scripts using it
+ # won't break.
+ parser.add_argument('-f', '--format', default='rest',
+ choices=['rest'],
+ help='The requested API protocol type (ignored)')
+ if 'hostname' in args:
+ help_text = ('Default application hostname, if none is specified '
+ 'for API service.')
+ parser.add_argument('--hostname', help=help_text)
+ if 'output' in args:
+ parser.add_argument('-o', '--output', default='.',
+ help='The directory to store output files')
+ if 'language' in args:
+ parser.add_argument('language',
+ help='The target output programming language')
+ if 'service' in args:
+ parser.add_argument('service', nargs='+',
+ help='Fully qualified service class name')
+ if 'discovery_doc' in args:
+ parser.add_argument('discovery_doc', nargs=1,
+ help='Path to the discovery document')
+ if 'build_system' in args:
+ parser.add_argument('-bs', '--build_system', default='default',
+ help='The target build system')
+
+ parser = _EndpointsParser(prog=prog)
+ subparsers = parser.add_subparsers(
+ title='subcommands', metavar='{%s}' % ', '.join(_VISIBLE_COMMANDS))
+
+ get_client_lib = subparsers.add_parser(
+ 'get_client_lib', help=('Generates discovery documents and client '
+ 'libraries from service classes'))
+ get_client_lib.set_defaults(callback=_GetClientLibCallback)
+ AddStandardOptions(get_client_lib, 'application', 'hostname', 'output',
+ 'language', 'service', 'build_system')
+
+ get_discovery_doc = subparsers.add_parser(
+ 'get_discovery_doc',
+ help='Generates discovery documents from service classes')
+ get_discovery_doc.set_defaults(callback=_GenDiscoveryDocCallback)
+ AddStandardOptions(get_discovery_doc, 'application', 'format', 'hostname',
+ 'output', 'service')
+
+ get_openapi_spec = subparsers.add_parser(
+ 'get_openapi_spec',
+ help='Generates OpenAPI (Swagger) specs from service classes')
+ get_openapi_spec.set_defaults(callback=_GenOpenApiSpecCallback)
+ AddStandardOptions(get_openapi_spec, 'application', 'hostname', 'output',
+ 'service')
+ get_openapi_spec.add_argument('--x-google-api-name', action='store_true',
+ help="Add the 'x-google-api-name' field to the generated spec")
+
+ # Create an alias for get_openapi_spec called get_swagger_spec to support
+ # the old-style naming. This won't be a visible command, but it will still
+ # function to support legacy scripts.
+ get_swagger_spec = subparsers.add_parser(
+ 'get_swagger_spec',
+ help='Generates OpenAPI (Swagger) specs from service classes')
+ get_swagger_spec.set_defaults(callback=_GenOpenApiSpecCallback)
+ AddStandardOptions(get_swagger_spec, 'application', 'hostname', 'output',
+ 'service')
+
+ # By removing the help attribute, the following three actions won't be
+ # displayed in usage message
+ gen_api_config = subparsers.add_parser('gen_api_config')
+ gen_api_config.set_defaults(callback=_GenApiConfigCallback)
+ AddStandardOptions(gen_api_config, 'application', 'hostname', 'output',
+ 'service')
+
+ gen_discovery_doc = subparsers.add_parser('gen_discovery_doc')
+ gen_discovery_doc.set_defaults(callback=_GenDiscoveryDocCallback)
+ AddStandardOptions(gen_discovery_doc, 'application', 'format', 'hostname',
+ 'output', 'service')
+
+ gen_client_lib = subparsers.add_parser('gen_client_lib')
+ gen_client_lib.set_defaults(callback=_GenClientLibCallback)
+ AddStandardOptions(gen_client_lib, 'output', 'language', 'discovery_doc',
+ 'build_system')
+
+ return parser
+
+
+def _SetupStubs():
+ tb = testbed.Testbed()
+ tb.setup_env(CURRENT_VERSION_ID='1.0')
+ tb.activate()
+ for k, v in testbed.INIT_STUB_METHOD_NAMES.items():
+ # The old stub initialization code didn't support the image service at all
+ # so we just ignore it here.
+ if k != 'images':
+ getattr(tb, v)()
+
+
+def main(argv):
+ logging.basicConfig()
+ # silence warnings from endpoints.apiserving; they're not relevant
+ # to command-line operation.
+ logging.getLogger('endpoints.apiserving').setLevel(logging.ERROR)
+
+ _SetupStubs()
+
+ parser = MakeParser(argv[0])
+ args = parser.parse_args(argv[1:])
+
+ # Handle the common "application" argument here, since most of the handlers
+ # use this.
+ application_path = getattr(args, 'application', None)
+ if application_path is not None:
+ sys.path.insert(0, os.path.abspath(application_path))
+
+ args.callback(args)